From b08b662354598d21799e187765590a9d268c368a Mon Sep 17 00:00:00 2001 From: WeebDataHoarder <57538841+WeebDataHoarder@users.noreply.github.com> Date: Thu, 26 Oct 2023 06:44:41 +0200 Subject: [PATCH] VFR aware Y4M decoder and encoders --- cli/encode-server/encode.go | 15 +++++- cli/encode-server/main.go | 12 ++++- cli/encode-utils/config.go | 8 ++- cli/timecodes/timecodes.go | 34 +++++++++++++ decoder/y4m/y4m.go | 44 ++++++++++++++++- encoder/libaom/libaom.go | 8 +-- encoder/libx264/libx264.go | 14 ++++-- frame/frame.go | 2 +- frame/stream.go | 10 ++++ go.mod | 1 + go.sum | 2 + utilities/ratio.go | 34 +++++++++++++ utilities/timecodes.go | 99 +++++++++++++++++++++++++++++++++++++ utilities/timecodes_test.go | 88 +++++++++++++++++++++++++++++++++ 14 files changed, 357 insertions(+), 14 deletions(-) create mode 100644 cli/timecodes/timecodes.go create mode 100644 utilities/timecodes.go create mode 100644 utilities/timecodes_test.go diff --git a/cli/encode-server/encode.go b/cli/encode-server/encode.go index cb359fe..709b33a 100644 --- a/cli/encode-server/encode.go +++ b/cli/encode-server/encode.go @@ -52,7 +52,13 @@ func encodeFromReader(reader io.ReadCloser, job *Job, w http.ResponseWriter) { job.Status.Read.Store(0) job.Status.Processed.Store(0) - decoder, err := y4m.NewDecoder(reader, nil) + settings := make(map[string]any) + + if len(job.Config.Timecodes) > 0 { + settings["timecodes"] = job.Config.Timecodes + } + + decoder, err := y4m.NewDecoder(reader, settings) if err != nil { w.Header().Set("x-encoder-error", "") w.Header().Set("x-decoder-error", err.Error()) @@ -61,7 +67,12 @@ func encodeFromReader(reader io.ReadCloser, job *Job, w http.ResponseWriter) { } //check decoder frame properties match - if decoder.Properties() != job.Config.Properties { + decProps := decoder.Properties() + if decProps.Width != job.Config.Properties.Width || + decProps.Height != job.Config.Properties.Height || + decProps.ColorSpace != job.Config.Properties.ColorSpace || + decProps.FullColorRange != job.Config.Properties.FullColorRange || + decProps.TimeBase() != job.Config.Properties.TimeBase() { w.Header().Set("x-encoder-error", "") w.Header().Set("x-decoder-error", "mismatched config properties") w.WriteHeader(http.StatusBadRequest) diff --git a/cli/encode-server/main.go b/cli/encode-server/main.go index 251e61a..6c02fd2 100644 --- a/cli/encode-server/main.go +++ b/cli/encode-server/main.go @@ -6,6 +6,7 @@ import ( encode_utils "git.gammaspectra.live/S.O.N.G/Ignite/cli/encode-utils" "git.gammaspectra.live/S.O.N.G/Ignite/encoder/libaom" "git.gammaspectra.live/S.O.N.G/Ignite/encoder/libx264" + "git.gammaspectra.live/S.O.N.G/Ignite/utilities" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/load" @@ -124,6 +125,15 @@ func main() { return } + if cfg.TimecodesV1 != "" { + cfg.Timecodes, err = utilities.ParseTimecodesV1(strings.NewReader(cfg.TimecodesV1)) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + } + if job := func() *Job { JobsMutex.Lock() defer JobsMutex.Unlock() @@ -361,7 +371,7 @@ func main() { }) s := http.Server{ - ReadTimeout: time.Second * 10, + ReadTimeout: 0, IdleTimeout: time.Second * 60, WriteTimeout: 0, Addr: *listenAddr, diff --git a/cli/encode-utils/config.go b/cli/encode-utils/config.go index 44dcd8e..ac269f3 100644 --- a/cli/encode-utils/config.go +++ b/cli/encode-utils/config.go @@ -1,6 +1,9 @@ package encode_utils -import "git.gammaspectra.live/S.O.N.G/Ignite/frame" +import ( + "git.gammaspectra.live/S.O.N.G/Ignite/frame" + "git.gammaspectra.live/S.O.N.G/Ignite/utilities" +) type JobConfig struct { Encoder struct { @@ -9,6 +12,9 @@ type JobConfig struct { } `json:"encoder" yaml:"encoder"` Properties frame.StreamProperties `json:"properties" yaml:"properties"` + + TimecodesV1 string `json:"timecodes_v1" yaml:"timecodes_v1"` + Timecodes utilities.Timecodes `json:"timecodes" yaml:"timecodes"` } const ( diff --git a/cli/timecodes/timecodes.go b/cli/timecodes/timecodes.go new file mode 100644 index 0000000..bd90d8f --- /dev/null +++ b/cli/timecodes/timecodes.go @@ -0,0 +1,34 @@ +package main + +import ( + "flag" + "fmt" + "git.gammaspectra.live/S.O.N.G/Ignite/utilities" + "os" +) + +func main() { + inputPath := flag.String("input", "", "Input timecodes file") + inputFormat := flag.String("format", "v1", "Input format. Supported: v1") + + flag.Parse() + + f, err := os.Open(*inputPath) + if err != nil { + panic(err) + } + defer f.Close() + + switch *inputFormat { + case "v1": + tc, err := utilities.ParseTimecodesV1(f) + if err != nil { + panic(err) + } + for _, e := range tc { + _, _ = fmt.Printf("%d\n", e) + } + default: + panic("unsupported format") + } +} diff --git a/decoder/y4m/y4m.go b/decoder/y4m/y4m.go index 8319c75..1f21068 100644 --- a/decoder/y4m/y4m.go +++ b/decoder/y4m/y4m.go @@ -5,6 +5,7 @@ import ( "fmt" "git.gammaspectra.live/S.O.N.G/Ignite/color" "git.gammaspectra.live/S.O.N.G/Ignite/frame" + "git.gammaspectra.live/S.O.N.G/Ignite/utilities" "github.com/ulikunitz/xz" "io" "runtime" @@ -25,6 +26,7 @@ type Decoder struct { frameStartOffset int64 frameCounter int + timecodes utilities.Timecodes bufPool sync.Pool } @@ -76,9 +78,42 @@ func NewDecoder(reader io.Reader, settings map[string]any) (*Decoder, error) { } } + if t, ok := settings["timecodes"]; ok { + if tc, ok := t.(utilities.Timecodes); ok { + s.timecodes = tc + } + } + + // Flagged VFR + if s.properties.FrameRate.Numerator == 0 && s.properties.FrameRate.Denominator == 0 && len(s.timecodes) == 0 { + return nil, errors.New("Y4M indicates VFR but timecodes is not set") + } + + if s.properties.PixelAspectRatio.Numerator == 0 && s.properties.PixelAspectRatio.Denominator == 0 { + // Assume 1:1 + s.properties.PixelAspectRatio = utilities.NewRatio(1, 1) + } + + s.properties.VFR = s.IsVFR() + return s, nil } +func (s *Decoder) IsVFR() bool { + return len(s.timecodes) != 0 +} + +// FramePTS Returns -1 if not found +func (s *Decoder) FramePTS(n int) int64 { + if s.IsVFR() { + if n >= len(s.timecodes) { + return -1 + } + return s.timecodes[n] + } + return int64(n) +} + func (s *Decoder) Properties() frame.StreamProperties { return s.properties } @@ -175,14 +210,19 @@ func (s *Decoder) GetFrame() (parameters map[Parameter][]string, frameObject fra return nil, nil, err } + pts := s.FramePTS(s.frameCounter) + if pts == -1 { + return nil, nil, fmt.Errorf("frame %d PTS could not be calculated", s.frameCounter) + } + if s.properties.ColorSpace.BitDepth > 8 { //it's copied below defer s.bufPool.Put(buf) - if frameObject, err = frame.NewUint16FrameFromBytes(s.properties.FrameProperties(), int64(s.frameCounter), buf); err != nil { + if frameObject, err = frame.NewUint16FrameFromBytes(s.properties.FrameProperties(), pts, buf); err != nil { return nil, nil, err } } else { - if f8, err := frame.NewUint8FrameFromBytes(s.properties.FrameProperties(), int64(s.frameCounter), buf); err != nil { + if f8, err := frame.NewUint8FrameFromBytes(s.properties.FrameProperties(), pts, buf); err != nil { s.bufPool.Put(buf) return nil, nil, err } else { diff --git a/encoder/libaom/libaom.go b/encoder/libaom/libaom.go index f66bf0e..a244672 100644 --- a/encoder/libaom/libaom.go +++ b/encoder/libaom/libaom.go @@ -131,10 +131,10 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str * \ref RECOMMENDED method is to set the timebase to that of the parent * container or multimedia framework (ex: 1/1000 for ms, as in FLV). */ - reciprocalFrameRate := properties.FrameRate.Reciprocal() + timeBase := properties.TimeBase() - e.cfg.g_timebase.num = C.int(reciprocalFrameRate.Numerator) - e.cfg.g_timebase.den = C.int(reciprocalFrameRate.Denominator) + e.cfg.g_timebase.num = C.int(timeBase.Numerator) + e.cfg.g_timebase.den = C.int(timeBase.Denominator) // boolean settings @@ -279,7 +279,7 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str } var err error - if e.w, err = obuwriter.NewWriter(w, properties.Width, properties.Height, 0x31305641, reciprocalFrameRate); err != nil { + if e.w, err = obuwriter.NewWriter(w, properties.Width, properties.Height, 0x31305641, timeBase); err != nil { return nil, err } diff --git a/encoder/libx264/libx264.go b/encoder/libx264/libx264.go index 7f4e2e8..56c6fb0 100644 --- a/encoder/libx264/libx264.go +++ b/encoder/libx264/libx264.go @@ -102,11 +102,19 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str e.params.i_width = C.int(properties.Width) e.params.i_height = C.int(properties.Height) - e.params.b_vfr_input = 0 + + if properties.VFR { + e.params.b_vfr_input = 1 + } else { + e.params.b_vfr_input = 0 + } e.params.b_repeat_headers = 1 e.params.b_annexb = 1 - e.params.i_fps_num = C.uint32_t(properties.FrameRate.Numerator) - e.params.i_fps_den = C.uint32_t(properties.FrameRate.Denominator) + + timeBase := properties.TimeBase() + + e.params.i_timebase_num = C.uint32_t(timeBase.Numerator) + e.params.i_timebase_den = C.uint32_t(timeBase.Denominator) if properties.FullColorRange { e.params.vui.b_fullrange = 1 } else { diff --git a/frame/frame.go b/frame/frame.go index 92a19a5..e8afa4e 100644 --- a/frame/frame.go +++ b/frame/frame.go @@ -11,7 +11,7 @@ type AllowedFrameTypes interface { type Frame interface { Properties() Properties - // PTS usually frame number + // PTS usually frame number, but can differ on VFR PTS() int64 // Get16 get a pixel sample in 16-bit depth diff --git a/frame/stream.go b/frame/stream.go index c7e3a80..d7856d6 100644 --- a/frame/stream.go +++ b/frame/stream.go @@ -32,6 +32,16 @@ type StreamProperties struct { ColorSpace color.Space `json:"colorspace" yaml:"colorspace"` FrameRate utilities.Ratio `json:"framerate" yaml:"framerate"` FullColorRange bool `json:"fullrange" yaml:"fullrange"` + VFR bool `json:"vfr" yaml:"vfr"` +} + +func (p StreamProperties) TimeBase() utilities.Ratio { + timeBase := p.FrameRate.Reciprocal() + if p.VFR { + timeBase = utilities.NewRatio(1, 1000) + } + + return timeBase } func (p StreamProperties) FrameProperties() Properties { diff --git a/go.mod b/go.mod index a3116be..b0f8e3a 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module git.gammaspectra.live/S.O.N.G/Ignite go 1.21 require ( + github.com/nethruster/go-fraction v0.0.0-20221224165113-1b5f693330ad github.com/shirou/gopsutil v3.21.11+incompatible github.com/stretchr/testify v1.8.1 github.com/ulikunitz/xz v0.5.11 diff --git a/go.sum b/go.sum index 3efb561..cc8da2d 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/nethruster/go-fraction v0.0.0-20221224165113-1b5f693330ad h1:HtuO+7iVoOXFaZouOdlIgT8gjN3lhk8Gfa2Eah34qSw= +github.com/nethruster/go-fraction v0.0.0-20221224165113-1b5f693330ad/go.mod h1:pyvrvZatpiWIxRlZpeEU0DD6SKOxgLMqZV/AU4Yz6V8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= diff --git a/utilities/ratio.go b/utilities/ratio.go index 9501bcd..f8839c4 100644 --- a/utilities/ratio.go +++ b/utilities/ratio.go @@ -3,6 +3,7 @@ package utilities import ( "fmt" "gopkg.in/yaml.v3" + "math" ) type Ratio struct { @@ -36,3 +37,36 @@ func (r Ratio) String() string { func (r Ratio) Reciprocal() Ratio { return Ratio{Numerator: r.Denominator, Denominator: r.Numerator} } + +const NTSCRatio = float64(1001) / 1000 + +const ratioEpsilon = 1.0e-9 + +func NewRatio(n, d int) Ratio { + return Ratio{ + Numerator: n, + Denominator: d, + } +} + +// FPSToRatio Attempt common FPS conversions to ratios +func FPSToRatio(fps float64) Ratio { + // common NTSC + ntscValue := fps * NTSCRatio + + // Number is whole NTSC + if math.Abs(ntscValue-math.Round(ntscValue)) < ratioEpsilon { + return NewRatio(int(math.Round(ntscValue))*1000, 1001) + } + + // Number is whole other + if math.Abs(fps-math.Round(fps)) < ratioEpsilon { + return NewRatio(int(math.Round(fps)), 1) + } + + // Get a few decimals of precision + return Ratio{ + Numerator: int(fps * 1000), + Denominator: 1000, + } +} diff --git a/utilities/timecodes.go b/utilities/timecodes.go new file mode 100644 index 0000000..73672c4 --- /dev/null +++ b/utilities/timecodes.go @@ -0,0 +1,99 @@ +package utilities + +import ( + "bufio" + "fmt" + "github.com/nethruster/go-fraction" + "io" + "strings" +) + +// Timecodes Entries in milliseconds +type Timecodes []int64 + +const timecodesv1Header = "# timecode format v1" + +func ParseTimecodesV1(reader io.Reader) (Timecodes, error) { + var tc Timecodes + + scanner := bufio.NewScanner(reader) + + // Read header + if !scanner.Scan() { + return nil, io.ErrUnexpectedEOF + } + switch strings.TrimSpace(scanner.Text()) { + case timecodesv1Header: + default: + return nil, fmt.Errorf("unexpected line: %s", strings.TrimSpace(scanner.Text())) + } + // Read assume line + + if !scanner.Scan() { + return nil, io.ErrUnexpectedEOF + } + + var fps float64 + + var from, to int64 + + _, err := fmt.Sscanf(strings.ToLower(strings.TrimSpace(scanner.Text())), "assume %f", &fps) + if err != nil { + return nil, err + } + + assumed := FPSToRatio(fps) + assumedFraction, err := fraction.New(assumed.Denominator, assumed.Numerator) + if err != nil { + return nil, err + } + if assumed.Denominator == 1000 { + assumedFraction, err = fraction.FromFloat64(1 / fps) + if err != nil { + return nil, err + } + } + + running, err := fraction.New(0, 1000) + if err != nil { + return nil, err + } + + var currentFrame int64 + + for scanner.Scan() { + line := strings.ReplaceAll(strings.TrimSpace(scanner.Text()), " ", "") + if line == "" || line[0] == '#' { + continue + } + + _, err := fmt.Sscanf(strings.ReplaceAll(strings.TrimSpace(scanner.Text()), " ", ""), "%d,%d,%f", &from, &to, &fps) + if err != nil { + return nil, err + } + + ratio := FPSToRatio(fps) + frac, err := fraction.New(ratio.Denominator, ratio.Numerator) + if err != nil { + return nil, err + } + if ratio.Denominator == 1000 { + frac, err = fraction.FromFloat64(1 / fps) + if err != nil { + return nil, err + } + } + + for ; currentFrame < from; currentFrame++ { + tc = append(tc, (running.Numerator()*1000)/running.Denominator()) + running = running.Add(assumedFraction) + } + + for ; currentFrame <= to; currentFrame++ { + tc = append(tc, (running.Numerator()*1000)/running.Denominator()) + running = running.Add(frac) + } + } + + return tc, nil +} diff --git a/utilities/timecodes_test.go b/utilities/timecodes_test.go new file mode 100644 index 0000000..fbfef06 --- /dev/null +++ b/utilities/timecodes_test.go @@ -0,0 +1,88 @@ +package utilities + +import ( + "bytes" + "runtime" + "testing" +) + +var tcTest = []byte(`# timecode format v1 +Assume 23.976023976024 +380,382,17.982017982018 +439,443,29.970029970030 +488,490,17.982017982018 +539,543,29.970029970030 +608,610,17.982017982018 +775,779,29.970029970030 +1284,1288,29.970029970030 +1481,1485,29.970029970030 +1726,1728,17.982017982018 +1985,1989,29.970029970030 +2034,2036,17.982017982018 +2153,2157,29.970029970030 +2226,2230,29.970029970030 +2419,2421,17.982017982018 +2486,2490,29.970029970030 +2891,2893,17.982017982018 +5818,5822,29.970029970030 +6407,6409,17.982017982018 +6458,6462,29.970029970030 +6715,6719,29.970029970030 +7276,7280,29.970029970030 +8873,8877,29.970029970030 +9066,9070,29.970029970030 +10083,10087,29.970029970030 +12096,12100,29.970029970030 +13077,13079,17.982017982018 +13392,13396,29.970029970030 +14213,14215,17.982017982018 +14228,14232,29.970029970030 +15941,15945,29.970029970030 +16894,16898,29.970029970030 +17079,17081,17.982017982018 +17214,17218,29.970029970030 +17255,17257,17.982017982018 +17462,17466,29.970029970030 +17691,17695,29.970029970030 +19804,19808,29.970029970030 +23665,23669,29.970029970030 +23890,23894,29.970029970030 +24411,24415,29.970029970030 +25572,25576,29.970029970030 +25637,25639,17.982017982018 +25744,25748,29.970029970030 +25809,25811,17.982017982018 +26100,26104,29.970029970030 +27533,27537,29.970029970030 +28610,28614,29.970029970030 +29099,29101,17.982017982018 +29258,29262,29.970029970030 +29619,29623,29.970029970030 +29776,29778,17.982017982018 +29995,29999,29.970029970030 +30164,30166,17.982017982018 +30191,30195,29.970029970030 +30752,30754,17.982017982018 +31995,31997,17.982017982018 +32546,32550,29.970029970030 +34243,34247,29.970029970030 +35580,35584,29.970029970030 +37581,37585,29.970029970030 +38778,38782,29.970029970030 +39735,39739,29.970029970030 +39956,39960,29.970029970030 +40121,40125,29.970029970030 +40322,40324,17.982017982018 +40349,40351,17.982017982018 +40372,40376,29.970029970030 +40677,47463,59.940059940059 +`) + +func TestParseTimecodesV1(t *testing.T) { + tc, err := ParseTimecodesV1(bytes.NewReader(tcTest)) + if err != nil { + t.Fatal(err) + } + + runtime.KeepAlive(tc) +}