Improved libaom/libx264, proper pool, frame stats

cli/encode-server/encode.go

@@ -1,3 +1,5 @@
+//go:build cgo && !disable_library_libaom && !disable_library_libx264
+
 package main
 
 import (
@@ -111,6 +113,7 @@ func encodeFromReader(reader io.ReadCloser, job *Job, w http.ResponseWriter) {
 			w.Header().Set("x-decoder-error", "")
 			return
 		}
+		f.Return()
 		job.Status.Processed.Add(1)
 		//log.Printf("[job %s] %d", job.Id, job.Status.Read.Load())
 	}

cli/encode-server/job.go

@@ -1,3 +1,5 @@
+//go:build cgo && !disable_library_libaom && !disable_library_libx264
+
 package main
 
 import (

cli/encode-server/main.go

@@ -1,3 +1,5 @@
+//go:build cgo && !disable_library_libaom && !disable_library_libx264
+
 package main
 
 import (

decoder/libdav1d/libdav1d.go

@@ -39,7 +39,7 @@ type Decoder struct {
 
 	closer sync.Once
 
-	bufPool sync.Pool
+	pool *frame.Pool
 }
 
 var dav1dVersion = fmt.Sprintf("dav1d %s", C.GoString(C.dav1d_version()))
@@ -174,7 +174,7 @@ const (
 	planeV = 2
 )
 
-func (d *Decoder) pictureToFrame() (frame.Frame, error) {
+func (d *Decoder) pictureToFrame() (f frame.Frame, err error) {
 
 	properties := d.properties.FrameProperties()
 
@@ -225,66 +225,29 @@ func (d *Decoder) pictureToFrame() (frame.Frame, error) {
 
 	defer C.dav1d_picture_unref(&d.picture)
 
-	if bitDepth > 8 { //16-bit
-
-		yData := unsafe.Slice((*byte)(d.picture.data[planeY]), properties.Height*properties.Width*2)
-		var uData, vData []byte
-
-		if d.picture.p.layout != C.DAV1D_PIXEL_LAYOUT_I400 {
-			uData = unsafe.Slice((*byte)(d.picture.data[planeU]), chromaHeight*chromaWidth*2)
-			vData = unsafe.Slice((*byte)(d.picture.data[planeV]), chromaHeight*chromaWidth*2)
-		}
-
-		n := len(yData) + len(uData) + len(vData)
-
-		var buf []byte
-		if b := d.bufPool.Get(); b != nil && len(b.([]byte)) == n {
-			buf = b.([]byte)
-		} else {
-			buf = make([]byte, n)
-		}
-
-		copy(buf, yData)
-		copy(buf[len(yData):], uData)
-		copy(buf[len(yData)+len(uData):], vData)
-
-		if f, err := frame.NewUint16FrameFromBytes(properties, int64(d.picture.m.timestamp), buf); err != nil {
+	if d.pool == nil || properties != d.pool.Properties() {
+		//initialize pool to known size
+		d.pool, err = frame.NewPool(properties)
+		if err != nil {
 			return nil, err
-		} else {
-			d.bufPool.Put(buf)
-			return f, nil
-		}
-	} else {
-		yData := unsafe.Slice((*byte)(d.picture.data[planeY]), properties.Height*properties.Width)
-		var uData, vData []byte
-
-		if d.picture.p.layout != C.DAV1D_PIXEL_LAYOUT_I400 {
-			uData = unsafe.Slice((*byte)(d.picture.data[planeU]), chromaHeight*chromaWidth)
-			vData = unsafe.Slice((*byte)(d.picture.data[planeV]), chromaHeight*chromaWidth)
-		}
-
-		n := len(yData) + len(uData) + len(vData)
-
-		var buf []byte
-		if b := d.bufPool.Get(); b != nil && len(b.([]byte)) == n {
-			buf = b.([]byte)
-		} else {
-			buf = make([]byte, n)
-		}
-
-		copy(buf, yData)
-		copy(buf[len(yData):], uData)
-		copy(buf[len(yData)+len(uData):], vData)
-
-		if f, err := frame.NewUint8FrameFromBytes(properties, int64(d.picture.m.timestamp), buf); err != nil {
-			return nil, err
-		} else {
-			runtime.SetFinalizer(f, func(frameUint8 *frame.FrameUint8) {
-				d.bufPool.Put(buf)
-			})
-			return f, nil
 		}
 	}
+
+	f = d.pool.Get(int64(d.picture.m.timestamp), int64(d.picture.m.timestamp)+int64(d.picture.m.duration))
+
+	var yData, uData, vData []byte
+
+	yData = unsafe.Slice((*byte)(d.picture.data[planeY]), properties.Height*properties.Width*(bitDepth/2))
+	if d.picture.p.layout != C.DAV1D_PIXEL_LAYOUT_I400 {
+		uData = unsafe.Slice((*byte)(d.picture.data[planeU]), chromaHeight*chromaWidth*(bitDepth/2))
+		vData = unsafe.Slice((*byte)(d.picture.data[planeV]), chromaHeight*chromaWidth*(bitDepth/2))
+	}
+
+	copy(f.GetLuma(), yData)
+	copy(f.GetCb(), uData)
+	copy(f.GetCr(), vData)
+
+	return f, nil
 }
 
 func (d *Decoder) DecodeStream() *frame.Stream {

decoder/libdav1d/libdav1d_test.go

@@ -35,6 +35,7 @@ func testDecode(sample testdata.TestSample, t *testing.T) {
 			if decoded%50 == 0 {
 				t.Logf("%d/%d", decoded, sample.Frames)
 			}
+			decodedFrame.Return()
 		}
 
 		if decoded != sample.Frames {

decoder/y4m/y4m.go

@@ -8,10 +8,8 @@ import (
 	"git.gammaspectra.live/S.O.N.G/Ignite/utilities"
 	"github.com/ulikunitz/xz"
 	"io"
-	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 )
 
 type Decoder struct {
@@ -28,7 +26,7 @@ type Decoder struct {
 	frameCounter int
 	timecodes    utilities.Timecodes
 
-	bufPool sync.Pool
+	pool *frame.Pool
 }
 
 type Parameter byte
@@ -59,19 +57,16 @@ func NewDecoder(reader io.Reader, settings map[string]any) (*Decoder, error) {
 		r:          reader,
 		parameters: make(map[Parameter][]string),
 	}
+	var err error
 
-	if err := s.readHeader(); err != nil {
+	if err = s.readHeader(); err != nil {
 		return nil, err
 	}
 
-	if err := s.parseParameters(); err != nil {
+	if err = s.parseParameters(); err != nil {
 		return nil, err
 	}
 
-	s.bufPool.New = func() any {
-		return make([]byte, s.frameSize)
-	}
-
 	if t, ok := settings["seek_table"]; ok {
 		if table, ok := t.([]int64); ok {
 			s.frameSeekTable = table
@@ -96,6 +91,11 @@ func NewDecoder(reader io.Reader, settings map[string]any) (*Decoder, error) {
 
 	s.properties.VFR = s.IsVFR()
 
+	s.pool, err = frame.NewPool(s.properties.FrameProperties())
+	if err != nil {
+		return nil, err
+	}
+
 	return s, nil
 }
 
@@ -204,34 +204,18 @@ func (s *Decoder) GetFrame() (parameters map[Parameter][]string, frameObject fra
 		s.frameSeekTable[s.frameCounter] = index
 	}
 
-	var buf []byte
-	if buf, err = s.readFrameData(); err != nil {
-		s.bufPool.Put(buf)
-		return nil, nil, err
-	}
-
 	pts := s.FramePTS(s.frameCounter)
 	if pts == -1 {
 		return nil, nil, fmt.Errorf("frame %d PTS could not be calculated", s.frameCounter)
 	}
 
-	if s.properties.ColorSpace.BitDepth > 8 {
-		//it's copied below
-		defer s.bufPool.Put(buf)
-		if frameObject, err = frame.NewUint16FrameFromBytes(s.properties.FrameProperties(), pts, buf); err != nil {
-			return nil, nil, err
-		}
-	} else {
-		if f8, err := frame.NewUint8FrameFromBytes(s.properties.FrameProperties(), pts, buf); err != nil {
-			s.bufPool.Put(buf)
-			return nil, nil, err
-		} else {
-			frameObject = f8
-			runtime.SetFinalizer(f8, func(f *frame.FrameUint8) {
-				//return buffer to pool once top frame is not in use
-				s.bufPool.Put(buf)
-			})
-		}
+	nextPts := s.FramePTS(s.frameCounter + 1)
+	if nextPts == -1 {
+		nextPts = pts + (pts - s.FramePTS(s.frameCounter-1))
+	}
+
+	if frameObject, err = s.readFrameData(pts, nextPts); err != nil {
+		return nil, nil, err
 	}
 
 	s.frameCounter++
@@ -316,11 +300,10 @@ func (s *Decoder) readFrameHeader() (parameters map[Parameter][]string, err erro
 	return parameters, nil
 }
 
-func (s *Decoder) readFrameData() (buf []byte, err error) {
-	//TODO: reuse buffers, maybe channel?
-	buf = s.bufPool.Get().([]byte)
-	_, err = io.ReadFull(s.r, buf)
-	return buf, err
+func (s *Decoder) readFrameData(pts, nextPts int64) (f frame.Frame, err error) {
+	f = s.pool.Get(pts, nextPts)
+	_, err = io.ReadFull(s.r, f.GetJoint())
+	return f, err
 }
 
 func (s *Decoder) parseParameters() (err error) {

decoder/y4m/y4m_test.go

@@ -1,8 +1,10 @@
 package y4m
 
 import (
+	"errors"
 	"git.gammaspectra.live/S.O.N.G/Ignite/frame"
 	"git.gammaspectra.live/S.O.N.G/Ignite/testdata"
+	"io"
 	"testing"
 )
 
@@ -29,6 +31,9 @@ func testDecode(sample testdata.TestSample, t *testing.T) {
 	} else {
 		defer y4m.Close()
 		decoded := 0
+		var lastPts int64
+
+		mod := sample.Frames / 5
 
 		var frameProperties frame.Properties
 		for decodedFrame := range y4m.DecodeStream().Channel() {
@@ -37,15 +42,26 @@ func testDecode(sample testdata.TestSample, t *testing.T) {
 			}
 			//ingest
 			decoded++
-			if decoded%50 == 0 {
+			if decoded%mod == 0 {
 				t.Logf("%d/%d", decoded, sample.Frames)
 			}
+			lastPts = decodedFrame.PTS()
+			decodedFrame.Return()
+		}
+		if decoded%mod != 0 {
+			t.Logf("%d/%d", decoded, sample.Frames)
 		}
 
 		if decoded != sample.Frames {
 			t.Fatalf("expected %d frames, got %d", sample.Frames, decoded)
 		}
 
+		duration := y4m.Properties().TimeBase().PTSToDuration(lastPts)
+
+		if y4m.Properties().TimeBase().PTSToDuration(lastPts) != sample.Duration() {
+			t.Fatalf("expected %s duration, got %s", sample.Duration(), duration)
+		}
+
 		if frameProperties.Width != sample.Width {
 			t.Fatalf("expected %d width, got %d", sample.Width, frameProperties.Width)
 		}
@@ -79,3 +95,94 @@ func TestDecode_YUV420_2160p60_10bit(t *testing.T) {
 func TestDecode_YUV420_360p24_8bit_xz(t *testing.T) {
 	testDecode(testdata.Y4M_Big_Buck_Bunny_360p24_YUV420_8bit, t)
 }
+
+func testBench(sample testdata.TestSample, b *testing.B) {
+	b.ReportAllocs()
+	reader, err := sample.Open(b)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer func() {
+		reader.Close()
+	}()
+
+	var y4m *Decoder
+
+	init := func() {
+		if y4m != nil {
+			y4m.Close()
+		}
+
+		if rsc, ok := reader.(io.ReadSeekCloser); ok {
+			_, err = rsc.Seek(0, io.SeekStart)
+			if err != nil {
+				b.Fatal(err)
+			}
+		} else {
+			reader.Close()
+			reader, err = sample.Open(b)
+			if err != nil {
+				b.Fatal(err)
+			}
+		}
+
+		switch sample.Type {
+		case "y4m":
+			y4m, err = NewDecoder(reader, nil)
+		case "y4m.xz":
+			y4m, err = NewXZCompressedDecoder(reader, nil)
+		default:
+			b.Fatal("unsupported sample type")
+		}
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	var bytesDecoded, framesDecoded int
+
+	init()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		decodedFrame, err := y4m.Decode()
+		if err != nil {
+			b.StopTimer()
+			if errors.Is(err, io.EOF) {
+				//reset to start
+				init()
+				b.StartTimer()
+				continue
+			} else {
+				b.Fatal(err)
+			}
+		}
+		framesDecoded++
+		bytesDecoded += len(decodedFrame.GetJoint())
+		b.SetBytes(int64(len(decodedFrame.GetJoint())))
+		decodedFrame.Return()
+	}
+	b.StopTimer()
+
+	y4m.Close()
+	b.ReportMetric(float64(framesDecoded)/b.Elapsed().Seconds(), "frames/s")
+}
+
+func BenchmarkDecode_YUV420_720p24_8bit(b *testing.B) {
+	testBench(testdata.Y4M_Sintel_Trailer_720p24_YUV420_8bit, b)
+}
+
+func BenchmarkDecode_YUV444_720p50_8bit(b *testing.B) {
+	testBench(testdata.Y4M_Ducks_Take_Off_720p50_YUV444_8bit, b)
+}
+
+func BenchmarkDecode_YUV422_720p50_8bit(b *testing.B) {
+	testBench(testdata.Y4M_Ducks_Take_Off_720p50_YUV422_8bit, b)
+}
+
+func BenchmarkDecode_YUV420_2160p60_10bit(b *testing.B) {
+	testBench(testdata.Y4M_Netflix_FoodMarket_2160p60_YUV420_10bit, b)
+}
+
+func BenchmarkDecode_YUV420_360p24_8bit_xz(b *testing.B) {
+	testBench(testdata.Y4M_Big_Buck_Bunny_360p24_YUV420_8bit, b)
+}

encoder/encoder.go

@@ -9,3 +9,8 @@ type Encoder interface {
 	Close()
 	Version() string
 }
+
+type EncoderWithStatistics interface {
+	Encoder
+	Statistics() frame.Statistics
+}

encoder/libaom/libaom.c

@@ -8,6 +8,10 @@ aom_codec_err_t aom_codec_control_uint(aom_codec_ctx_t *ctx, int ctrl_id, unsign
     return aom_codec_control(ctx, ctrl_id, v);
 }
 
+aom_codec_err_t aom_codec_control_intptr(aom_codec_ctx_t *ctx, int ctrl_id, int* v) {
+    return aom_codec_control(ctx, ctrl_id, v);
+}
+
 void* aom_get_pkt_buf(aom_codec_cx_pkt_t *pkt){
     return pkt->data.frame.buf;
 }
@@ -16,6 +20,10 @@ size_t aom_get_pkt_sz(aom_codec_cx_pkt_t *pkt){
   return pkt->data.frame.sz;
 }
 
+int aom_get_pkt_partition_id(aom_codec_cx_pkt_t *pkt){
+  return pkt->data.frame.partition_id;
+}
+
 aom_codec_pts_t aom_get_pkt_pts(aom_codec_cx_pkt_t *pkt){
     return pkt->data.frame.pts;
 }
@@ -23,3 +31,7 @@ aom_codec_pts_t aom_get_pkt_pts(aom_codec_cx_pkt_t *pkt){
 aom_codec_frame_flags_t aom_get_pkt_flags(aom_codec_cx_pkt_t *pkt){
   return pkt->data.frame.flags;
 }
+
+const struct aom_codec_enc_cfg* aom_get_ctx_enc_cfg(aom_codec_ctx_t *ctx){
+  return ctx->config.enc;
+}

encoder/libaom/libaom.go

@@ -10,10 +10,12 @@ import "C"
 import (
 	"errors"
 	"fmt"
+	"git.gammaspectra.live/S.O.N.G/Ignite/color"
 	"git.gammaspectra.live/S.O.N.G/Ignite/frame"
 	"git.gammaspectra.live/S.O.N.G/Ignite/utilities/obuwriter"
 	"golang.org/x/exp/constraints"
 	"io"
+	"log"
 	"maps"
 	"runtime"
 	"strconv"
@@ -22,16 +24,18 @@ import (
 )
 
 type Encoder struct {
-	w              *obuwriter.Writer
-	cleaned        atomic.Bool
-	cfg            C.aom_codec_enc_cfg_t
-	codec          C.aom_codec_ctx_t
-	raw            *C.aom_image_t
-	frames         uint32
-	resourcePinner runtime.Pinner
+	w                   *obuwriter.Writer
+	cleaned             atomic.Bool
+	cfg                 C.aom_codec_enc_cfg_t
+	codec               C.aom_codec_ctx_t
+	raw                 *C.aom_image_t
+	rawBuffer           []byte
+	framesIn, framesOut int
+	frameStatistics     frame.FrameStatistics
+	resourcePinner      runtime.Pinner
 }
 
-var libaomVersion = "libaom-av1 " + C.GoString(C.aom_codec_version_str())
+var libaomVersion = "libaom-av1 " + C.GoString(C.aom_codec_version_str()) + " ABI " + strconv.FormatUint(C.AOM_ENCODER_ABI_VERSION, 10)
 
 func Version() string {
 	return libaomVersion
@@ -44,7 +48,9 @@ const (
 )
 
 func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[string]any) (*Encoder, error) {
-	e := &Encoder{}
+	e := &Encoder{
+		frameStatistics: make(frame.FrameStatistics),
+	}
 
 	clonedSettings := maps.Clone(settings)
 
@@ -74,21 +80,56 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 		return nil, errors.New("failed to get default codec config")
 	}
 
+	var bitsPerSample, decH, decV C.int
+
 	switch true {
 	case properties.ColorSpace.ChromaSampling.J == 4 && properties.ColorSpace.ChromaSampling.A == 4 && properties.ColorSpace.ChromaSampling.B == 4:
 		imageFormat = C.AOM_IMG_FMT_I444
 		e.cfg.g_profile = 1
+		switch properties.ColorSpace.BitDepth {
+		case C.AOM_BITS_8:
+			bitsPerSample = 24
+		case C.AOM_BITS_10:
+			bitsPerSample = 30
+		case C.AOM_BITS_12:
+			bitsPerSample = 36
+		}
+		decH = 1
+		decV = 1
 	case properties.ColorSpace.ChromaSampling.J == 4 && properties.ColorSpace.ChromaSampling.A == 2 && properties.ColorSpace.ChromaSampling.B == 2:
 		imageFormat = C.AOM_IMG_FMT_I422
 		e.cfg.g_profile = 2
+		switch properties.ColorSpace.BitDepth {
+		case C.AOM_BITS_8:
+			bitsPerSample = 16
+		case C.AOM_BITS_10:
+			bitsPerSample = 20
+		case C.AOM_BITS_12:
+			bitsPerSample = 24
+		}
+		decH = 2
+		decV = 1
 	case properties.ColorSpace.ChromaSampling.J == 4 && properties.ColorSpace.ChromaSampling.A == 2 && properties.ColorSpace.ChromaSampling.B == 0:
 		imageFormat = C.AOM_IMG_FMT_I420
 		e.cfg.g_profile = 0
+		switch properties.ColorSpace.BitDepth {
+		case C.AOM_BITS_8:
+			bitsPerSample = 12
+		case C.AOM_BITS_10:
+			bitsPerSample = 15
+		case C.AOM_BITS_12:
+			bitsPerSample = 18
+		}
+		decH = 2
+		decV = 2
 	case properties.ColorSpace.ChromaSampling.J == 4 && properties.ColorSpace.ChromaSampling.A == 0 && properties.ColorSpace.ChromaSampling.B == 0:
 		//mono is defined as 4:2:0, but monochrome is set on config
 		imageFormat = C.AOM_IMG_FMT_I420
 		e.cfg.g_profile = 0
 		e.cfg.monochrome = 1
+		bitsPerSample = C.int(properties.ColorSpace.BitDepth)
+		decH = 0
+		decV = 2
 	default:
 		return nil, errors.New("unsupported input chroma subsampling")
 
@@ -97,31 +138,95 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 	e.cfg.g_input_bit_depth = C.uint(properties.ColorSpace.BitDepth)
 	e.cfg.g_bit_depth = C.aom_bit_depth_t(properties.ColorSpace.BitDepth)
 
-	if e.cfg.g_bit_depth >= 12 { //only bitdepths up to 12 are supported, see aom_bit_depth_t
-		e.cfg.g_bit_depth = 12
+	if e.cfg.g_bit_depth >= C.AOM_BITS_12 { //only bitdepths up to 12 are supported, see aom_bit_depth_t
+		e.cfg.g_bit_depth = C.AOM_BITS_12
 		e.cfg.g_profile = 2
 	}
 
-	if e.cfg.g_input_bit_depth > 8 {
+	if e.cfg.g_input_bit_depth > C.AOM_BITS_8 {
 		imageFormat |= C.AOM_IMG_FMT_HIGHBITDEPTH
 	}
-	if e.cfg.g_bit_depth > 8 {
+	if e.cfg.g_bit_depth > C.AOM_BITS_8 {
 		flags |= C.AOM_CODEC_USE_HIGHBITDEPTH
 	}
 
-	e.raw = &C.aom_image_t{}
-	e.resourcePinner.Pin(e.raw)
-	if C.aom_img_alloc(e.raw, imageFormat, C.uint(properties.Width), C.uint(properties.Height), 1) == nil {
-		return nil, errors.New("failed to allocate image")
+	frameSize, err := properties.ColorSpace.FrameSize(properties.Width, properties.Height)
+	if err != nil {
+		return nil, err
 	}
 
+	bytesPerSample := 1
+	if properties.ColorSpace.BitDepth > C.AOM_BITS_8 {
+		bytesPerSample = 2
+	}
+
+	e.raw = &C.aom_image_t{}
+
+	e.raw.bit_depth = C.uint(properties.ColorSpace.BitDepth)
+	//todo: color primaries
+	e.raw.cp = C.AOM_CICP_CP_UNSPECIFIED
+	//todo: transfer characteristics
+	e.raw.tc = C.AOM_CICP_TC_UNSPECIFIED
+	//todo: matrix coefficients
+	e.raw.mc = C.AOM_CICP_MC_UNSPECIFIED
+	e.raw.monochrome = C.int(e.cfg.monochrome)
+	e.raw.fmt = imageFormat
+
+	switch properties.ColorSpace.ChromaSamplePosition {
+	case color.ChromaSamplePositionLeft:
+		e.raw.csp = C.AOM_CSP_VERTICAL
+	case color.ChromaSamplePositionCenter:
+		e.raw.csp = C.AOM_CSP_UNKNOWN
+	case color.ChromaSamplePositionTopLeft:
+		e.raw.csp = C.AOM_CSP_COLOCATED
+	default:
+		e.raw.csp = C.AOM_CSP_UNKNOWN
+	}
+
+	if properties.FullColorRange {
+		e.raw._range = C.AOM_CR_FULL_RANGE
+	} else {
+		e.raw._range = C.AOM_CR_STUDIO_RANGE
+	}
+
+	e.cfg.g_w = C.uint(properties.Width)
+	e.cfg.g_h = C.uint(properties.Height)
+
+	e.raw.w = e.cfg.g_w
+	e.raw.d_w = e.cfg.g_w
+	e.raw.h = e.cfg.g_h
+	e.raw.d_h = e.cfg.g_h
+	e.raw.bps = bitsPerSample
+	e.raw.x_chroma_shift = C.uint(decH >> 1)
+	e.raw.y_chroma_shift = C.uint(decV >> 1)
+
+	c_w := (C.int(properties.Width) + decH - 1) / decH
+	c_w *= C.int(bytesPerSample)
+
+	e.rawBuffer = make([]byte, frameSize)
+	e.resourcePinner.Pin(e.raw)
+	e.resourcePinner.Pin(unsafe.SliceData(e.rawBuffer))
+
+	iY := properties.ColorSpace.ChromaSampling.PlaneLumaSamples(properties.Width, properties.Height)
+	iCb := properties.ColorSpace.ChromaSampling.PlaneCbSamples(properties.Width, properties.Height)
+	iCr := properties.ColorSpace.ChromaSampling.PlaneCrSamples(properties.Width, properties.Height)
+
+	iY *= bytesPerSample
+	iCb *= bytesPerSample
+	iCr *= bytesPerSample
+
+	e.raw.stride[C.AOM_PLANE_Y] = C.int(properties.Width * bytesPerSample)
+	e.raw.stride[C.AOM_PLANE_U] = c_w
+	e.raw.stride[C.AOM_PLANE_V] = e.raw.stride[C.AOM_PLANE_U]
+
+	e.raw.planes[C.AOM_PLANE_Y] = (*C.uchar)(unsafe.Pointer(unsafe.SliceData(e.rawBuffer[:iY])))
+	e.raw.planes[C.AOM_PLANE_U] = (*C.uchar)(unsafe.Pointer(unsafe.SliceData(e.rawBuffer[iY : iY+iCb])))
+	e.raw.planes[C.AOM_PLANE_V] = (*C.uchar)(unsafe.Pointer(unsafe.SliceData(e.rawBuffer[iY+iCb : iY+iCb+iCr])))
+
 	runtime.SetFinalizer(e, func(encoder *Encoder) {
 		encoder.Close()
 	})
 
-	e.cfg.g_w = C.uint(properties.Width)
-	e.cfg.g_h = C.uint(properties.Height)
-
 	/*!\brief Stream timebase units
 	 *
 	 * Indicates the smallest interval of time, in seconds, used by the stream.
@@ -199,6 +304,8 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 		{&e.cfg.kf_max_dist, "kf-max-dist"},
 		{&e.cfg.sframe_dist, "sframe-dist"},
 		{&e.cfg.sframe_mode, "sframe-mode"},
+
+		{&e.cfg.use_fixed_qp_offsets, "use-fixed-qp-offsets"},
 	} {
 		//todo: unset setting from map
 		*s.p = C.uint(getSettingUnsigned(clonedSettings, s.n, uint(*s.p)))
@@ -224,19 +331,51 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 	//TODO: find all settings not set on AV1 encoder and place them on e.cfg
 
 	if aomErr = C.aom_codec_enc_init_ver(&e.codec, encoder, &e.cfg, flags, C.AOM_ENCODER_ABI_VERSION); aomErr != 0 {
-		return nil, fmt.Errorf("failed to initialize encoder: %s", C.GoString(e.codec.err_detail))
+		return nil, fmt.Errorf("failed to initialize encoder: err %d %s", aomErr, C.GoString(e.codec.err_detail))
 	}
 
 	if properties.FullColorRange {
-		if aomErr = C.aom_codec_control_uint(&e.codec, C.AV1E_SET_COLOR_RANGE, 1); aomErr != 0 {
+		if aomErr = C.aom_codec_control_uint(&e.codec, C.AV1E_SET_COLOR_RANGE, C.AOM_CR_FULL_RANGE); aomErr != 0 {
 			return nil, fmt.Errorf("failed to set color range")
 		}
 	} else {
-		if aomErr = C.aom_codec_control_uint(&e.codec, C.AV1E_SET_COLOR_RANGE, 0); aomErr != 0 {
+		if aomErr = C.aom_codec_control_uint(&e.codec, C.AV1E_SET_COLOR_RANGE, C.AOM_CR_STUDIO_RANGE); aomErr != 0 {
 			return nil, fmt.Errorf("failed to set color range")
 		}
 	}
 
+	//??? aomenc does this
+	if properties.ColorSpace.BitDepth == 12 {
+		if aomErr = C.aom_codec_control_uint(&e.codec, C.AV1E_SET_CHROMA_SUBSAMPLING_X, C.uint(decH>>1)); aomErr != 0 {
+			return nil, fmt.Errorf("failed to set chroma subsampling X")
+		}
+		if aomErr = C.aom_codec_control_uint(&e.codec, C.AV1E_SET_CHROMA_SUBSAMPLING_Y, C.uint(decV>>1)); aomErr != 0 {
+			return nil, fmt.Errorf("failed to set chroma subsampling Y")
+		}
+	}
+
+	//TODO: fill all
+	controlSettings := map[string]C.int{
+		"cpu-used":     C.AOME_SET_CPUUSED,
+		"auto-alt-ref": C.AOME_SET_ENABLEAUTOALTREF,
+		"sharpness":    C.AOME_SET_SHARPNESS,
+		"row-mt":       C.AV1E_SET_ROW_MT,
+		//"fp-mt": C.AV1E_SET_FP_MT,
+		"tile-columns":         C.AV1E_SET_TILE_COLUMNS,
+		"tile-rows":            C.AV1E_SET_TILE_ROWS,
+		"cq-level":             C.AOME_SET_CQ_LEVEL,
+		"max-reference-frames": C.AV1E_SET_MAX_REFERENCE_FRAMES,
+	}
+
+	for key, controlKey := range controlSettings {
+		if _, ok := clonedSettings[key]; ok {
+			val := getSettingUnsigned[uint](clonedSettings, key, 0)
+			if aomErr = C.aom_codec_control_uint(&e.codec, controlKey, C.uint(val)); aomErr != 0 {
+				return nil, fmt.Errorf("error setting parameter %s: %s", key, C.GoString(C.aom_codec_error_detail(&e.codec)))
+			}
+		}
+	}
+
 	for k, v := range clonedSettings {
 		if err := func() error {
 			var strVal *C.char
@@ -281,7 +420,6 @@ func NewEncoder(w io.Writer, properties frame.StreamProperties, settings map[str
 		}
 	}
 
-	var err error
 	if e.w, err = obuwriter.NewWriter(w, properties.Width, properties.Height, 0x31305641, timeBase); err != nil {
 		return nil, err
 	}
@@ -299,40 +437,27 @@ func (e *Encoder) EncodeStream(stream *frame.Stream) error {
 }
 
 func (e *Encoder) Encode(f frame.Frame) error {
+	/*luma := f.GetLuma()
+	cb := f.GetCb()
+	cr := f.GetCr()
+	copy(unsafe.Slice((*byte)(e.raw.planes[C.AOM_PLANE_Y]), len(luma)), luma)
+	copy(unsafe.Slice((*byte)(e.raw.planes[C.AOM_PLANE_U]), len(cb)), cb)
+	copy(unsafe.Slice((*byte)(e.raw.planes[C.AOM_PLANE_V]), len(cr)), cr)
+	*/
+	copy(e.rawBuffer, f.GetJoint())
 
-	switch typedFrame := f.(type) {
-	case frame.TypedFrame[uint8]:
-		luma := typedFrame.GetNativeLuma()
-		cb := typedFrame.GetNativeCb()
-		cr := typedFrame.GetNativeCr()
-		copy(unsafe.Slice((*byte)(e.raw.planes[0]), len(luma)), luma)
-		copy(unsafe.Slice((*byte)(e.raw.planes[1]), len(cb)), cb)
-		copy(unsafe.Slice((*byte)(e.raw.planes[2]), len(cr)), cr)
-	case frame.TypedFrame[uint16]:
-		luma := typedFrame.GetNativeLuma()
-		cb := typedFrame.GetNativeCb()
-		cr := typedFrame.GetNativeCr()
-		copy(unsafe.Slice((*uint16)(unsafe.Pointer(e.raw.planes[0])), len(luma)), luma)
-		copy(unsafe.Slice((*uint16)(unsafe.Pointer(e.raw.planes[1])), len(cb)), cb)
-		copy(unsafe.Slice((*uint16)(unsafe.Pointer(e.raw.planes[2])), len(cr)), cr)
-	default:
-		return errors.New("unknown frame type")
-	}
-
-	if _, err := e.encodeFrame(f.PTS(), e.raw); err != nil {
+	if _, err := e.encodeFrame(f.PTS(), f.NextPTS(), e.raw); err != nil {
 		return err
 	}
-
-	e.frames++
+	runtime.KeepAlive(f)
 
 	return nil
 }
 
-func (e *Encoder) encodeFrame(pts int64, raw *C.aom_image_t) (pkts int, err error) {
-	//TODO: make this a Source channel
+func (e *Encoder) encodeFrame(pts, nextPts int64, raw *C.aom_image_t) (pkts int, err error) {
 	var aomErr C.aom_codec_err_t
 
-	if aomErr = C.aom_codec_encode(&e.codec, raw, C.long(pts), 1, 0); aomErr != C.AOM_CODEC_OK {
+	if aomErr = C.aom_codec_encode(&e.codec, raw, C.long(pts), C.ulong(nextPts-pts), 0); aomErr != C.AOM_CODEC_OK {
 		if aomErr == C.AOM_CODEC_INCAPABLE {
 			return 0, errors.New("error encoding frame: AOM_CODEC_INCAPABLE")
 		} else if aomErr == C.AOM_CODEC_INVALID_PARAM {
@@ -344,6 +469,17 @@ func (e *Encoder) encodeFrame(pts int64, raw *C.aom_image_t) (pkts int, err erro
 		}
 	}
 
+	if raw != nil {
+		e.framesIn++
+	}
+
+	var quant64 C.int
+	if e.framesIn >= int(e.cfg.g_lag_in_frames) {
+		if aomErr = C.aom_codec_control_intptr(&e.codec, C.AOME_GET_LAST_QUANTIZER_64, &quant64); aomErr != C.AOM_CODEC_OK {
+			return 0, errors.New("error getting LAST_QUANTIZER_64")
+		}
+	}
+
 	var iter C.aom_codec_iter_t
 
 	for {
@@ -354,21 +490,63 @@ func (e *Encoder) encodeFrame(pts int64, raw *C.aom_image_t) (pkts int, err erro
 		pkts++
 
 		if pkt.kind == C.AOM_CODEC_CX_FRAME_PKT {
-			if err = e.w.WriteFrameBytes(uint64(C.aom_get_pkt_pts(pkt)), unsafe.Slice((*byte)(C.aom_get_pkt_buf(pkt)), int(C.aom_get_pkt_sz(pkt)))); err != nil {
+			partitionId := int(C.aom_get_pkt_partition_id(pkt))
+			if partitionId > 0 {
+				return 0, errors.New("partition id not supported")
+			}
+			packetPts := uint64(C.aom_get_pkt_pts(pkt))
+			buf := unsafe.Slice((*byte)(C.aom_get_pkt_buf(pkt)), int(C.aom_get_pkt_sz(pkt)))
+			flags := C.aom_get_pkt_flags(pkt)
+
+			e.frameStatistics[frame.FrameStatisticsKeyNumber] = e.framesOut
+			e.frameStatistics[frame.FrameStatisticsKeyPts] = int64(packetPts)
+			e.frameStatistics[frame.FrameStatisticsKeyQuantizer] = int(quant64)
+			e.frameStatistics[frame.FrameStatisticsKeySize] = len(buf)
+			e.frameStatistics[frame.FrameStatisticsKeyIsKeyFrame] = false
+			e.frameStatistics[frame.FrameStatisticsKeyIsIntraFrame] = false
+
+			if flags&C.AOM_FRAME_IS_KEY > 0 {
+				e.frameStatistics[frame.FrameStatisticsKeyIsKeyFrame] = true
+			}
+			if flags&C.AOM_FRAME_IS_INTRAONLY > 0 {
+				e.frameStatistics[frame.FrameStatisticsKeyIsIntraFrame] = true
+			}
+			if err = e.w.WriteFrameBytes(packetPts, buf); err != nil {
 				return pkts, err
 			}
+
+			e.framesOut++
+			runtime.KeepAlive(buf)
+		} else {
+
+			log.Printf("kind %d", pkt.kind)
 		}
 	}
 
 	return pkts, nil
 }
 
+func (e *Encoder) Statistics() frame.Statistics {
+	if len(e.frameStatistics) > 0 {
+		return frame.Statistics{
+			frame.StatisticsKeyFramesIn:     e.framesIn,
+			frame.StatisticsKeyFramesOut:    e.framesOut,
+			frame.StatisticsKeyLastFrameOut: maps.Clone(e.frameStatistics),
+		}
+	} else {
+		return frame.Statistics{
+			frame.StatisticsKeyFramesIn:  e.framesIn,
+			frame.StatisticsKeyFramesOut: e.framesOut,
+		}
+	}
+}
+
 func (e *Encoder) Flush() error {
 
 	var pkts int
 	var err error
 	for {
-		if pkts, err = e.encodeFrame(-1, nil); err != nil {
+		if pkts, err = e.encodeFrame(-1, 0, nil); err != nil {
 			return err
 		}
 		if pkts == 0 {
@@ -376,19 +554,21 @@ func (e *Encoder) Flush() error {
 		}
 	}
 
-	_ = e.w.WriteLength(e.frames)
+	_ = e.w.WriteLength(uint32(e.framesIn))
 
 	return nil
 }
 
 func (e *Encoder) Close() {
 	if e.cleaned.Swap(true) == false {
+		e.resourcePinner.Unpin()
 		if e.raw != nil {
-			C.aom_img_free(e.raw)
 			e.raw = nil
 		}
+		if e.rawBuffer != nil {
+			e.rawBuffer = nil
+		}
 		C.aom_codec_destroy(&e.codec)
-		e.resourcePinner.Unpin()
 	}
 }
 

encoder/libaom/libaom.h

@@ -9,7 +9,11 @@ aom_codec_err_t aom_codec_control_int(aom_codec_ctx_t *ctx, int ctrl_id, int v);
 
 aom_codec_err_t aom_codec_control_uint(aom_codec_ctx_t *ctx, int ctrl_id, unsigned int v);
 
+aom_codec_err_t aom_codec_control_intptr(aom_codec_ctx_t *ctx, int ctrl_id, int* v);
+
 void* aom_get_pkt_buf(aom_codec_cx_pkt_t *pkt);
 size_t aom_get_pkt_sz(aom_codec_cx_pkt_t *pkt);
 aom_codec_pts_t aom_get_pkt_pts(aom_codec_cx_pkt_t *pkt);
-aom_codec_frame_flags_t aom_get_pkt_flags(aom_codec_cx_pkt_t *pkt);
+int aom_get_pkt_partition_id(aom_codec_cx_pkt_t *pkt);
+aom_codec_frame_flags_t aom_get_pkt_flags(aom_codec_cx_pkt_t *pkt);
+const struct aom_codec_enc_cfg* aom_get_ctx_enc_cfg(aom_codec_ctx_t *ctx);

encoder/libx264/libx264.go

@@ -223,24 +223,12 @@ func (e *Encoder) EncodeStream(stream *frame.Stream) error {
 }
 
 func (e *Encoder) Encode(f frame.Frame) error {
-	switch typedFrame := f.(type) {
-	case frame.TypedFrame[uint8]:
-		luma := typedFrame.GetNativeLuma()
-		cb := typedFrame.GetNativeCb()
-		cr := typedFrame.GetNativeCr()
-		copy(unsafe.Slice((*byte)(e.pictureIn.img.plane[0]), len(luma)), luma)
-		copy(unsafe.Slice((*byte)(e.pictureIn.img.plane[1]), len(cb)), cb)
-		copy(unsafe.Slice((*byte)(e.pictureIn.img.plane[2]), len(cr)), cr)
-	case frame.TypedFrame[uint16]:
-		luma := typedFrame.GetNativeLuma()
-		cb := typedFrame.GetNativeCb()
-		cr := typedFrame.GetNativeCr()
-		copy(unsafe.Slice((*uint16)(unsafe.Pointer(e.pictureIn.img.plane[0])), len(luma)), luma)
-		copy(unsafe.Slice((*uint16)(unsafe.Pointer(e.pictureIn.img.plane[1])), len(cb)), cb)
-		copy(unsafe.Slice((*uint16)(unsafe.Pointer(e.pictureIn.img.plane[2])), len(cr)), cr)
-	default:
-		return errors.New("unknown frame type")
-	}
+	luma := f.GetLuma()
+	cb := f.GetCb()
+	cr := f.GetCr()
+	copy(unsafe.Slice((*byte)(e.pictureIn.img.plane[0]), len(luma)), luma)
+	copy(unsafe.Slice((*byte)(e.pictureIn.img.plane[1]), len(cb)), cb)
+	copy(unsafe.Slice((*byte)(e.pictureIn.img.plane[2]), len(cr)), cr)
 
 	e.pictureIn.i_pts = C.int64_t(f.PTS())
 

frame/frame.go

@@ -13,12 +13,24 @@ type Frame interface {
 	Properties() Properties
 	// PTS usually frame number, but can differ on VFR
 	PTS() int64
+	// NextPTS Next PTS
+	NextPTS() int64
 
 	// Get16 get a pixel sample in 16-bit depth
 	Get16(x, y int) (Y uint16, Cb uint16, Cr uint16)
 
 	// Get8 get a pixel sample in 8-bit depth
 	Get8(x, y int) (Y uint8, Cb uint8, Cr uint8)
+
+	GetLuma() []byte
+	GetCb() []byte
+	GetCr() []byte
+	// GetJoint Gets Luma+Cb+Cr slice. Do not keep references to this slice, copy instead.
+	// This is unsafe
+	GetJoint() []byte
+
+	// Return Finishes using this frame and marks it for reuse
+	Return()
 }
 
 type TypedFrame[T AllowedFrameTypes] interface {
@@ -33,6 +45,12 @@ type TypedFrame[T AllowedFrameTypes] interface {
 	GetNativeCb() []T
 	// GetNativeCr also known as V. Do not keep references to this slice, copy instead.
 	GetNativeCr() []T
+	// GetNativeJoint Gets Luma+Cb+Cr slice. Do not keep references to this slice, copy instead.
+	GetNativeJoint() []T
+
+	FillNativeLuma([]T)
+	FillNativeCb([]T)
+	FillNativeCr([]T)
 }
 
 type Properties struct {

frame/frame_uint16.go

@@ -1,66 +1,44 @@
 package frame
 
 import (
-	"errors"
-	"runtime"
 	"unsafe"
 )
 
-type FrameUint16 struct {
+type fUint16 struct {
 	properties Properties
+	ret        func(f Frame)
 	Pts        int64
+	NextPts    int64
 	Y          []uint16
 	Cb         []uint16
 	Cr         []uint16
 }
 
-func NewUint16FrameFromBytes(properties Properties, pts int64, data []byte) (*FrameUint16, error) {
-	if frameLength, _ := properties.ColorSpace.FrameSize(properties.Width, properties.Height); frameLength != len(data) {
-		return nil, errors.New("wrong length of data")
-	}
-
-	if properties.ColorSpace.BitDepth >= 16 {
-		return nil, errors.New("wrong bit depth")
-	}
-
-	buf := make([]uint16, len(data)/2)
-	copy(buf, unsafe.Slice((*uint16)(unsafe.Pointer(unsafe.SliceData(data))), len(data)/2))
-	runtime.KeepAlive(data)
-
-	iY := properties.ColorSpace.ChromaSampling.PlaneLumaSamples(properties.Width, properties.Height)
-	iCb := properties.ColorSpace.ChromaSampling.PlaneCbSamples(properties.Width, properties.Height)
-	iCr := properties.ColorSpace.ChromaSampling.PlaneCrSamples(properties.Width, properties.Height)
-
-	return &FrameUint16{
-		properties: properties,
-		Y:          buf[:iY],
-		Cb:         buf[iY : iY+iCb],
-		Cr:         buf[iY+iCb : iY+iCb+iCr],
-		Pts:        pts,
-	}, nil
-}
-
-func (i *FrameUint16) Get16(x, y int) (Y uint16, Cb uint16, Cr uint16) {
+func (i *fUint16) Get16(x, y int) (Y uint16, Cb uint16, Cr uint16) {
 	cy, cb, cr := i.GetNative(x, y)
 
 	return cy << (16 - i.properties.ColorSpace.BitDepth), cb << (16 - i.properties.ColorSpace.BitDepth), cr << (16 - i.properties.ColorSpace.BitDepth)
 }
 
-func (i *FrameUint16) Get8(x, y int) (Y uint8, Cb uint8, Cr uint8) {
+func (i *fUint16) Get8(x, y int) (Y uint8, Cb uint8, Cr uint8) {
 	cy, cb, cr := i.GetNative(x, y)
 
 	return uint8(cy >> (i.properties.ColorSpace.BitDepth - 8)), uint8(cb >> (i.properties.ColorSpace.BitDepth - 8)), uint8(cr >> (i.properties.ColorSpace.BitDepth - 8))
 }
 
-func (i *FrameUint16) Properties() Properties {
+func (i *fUint16) Properties() Properties {
 	return i.properties
 }
 
-func (i *FrameUint16) PTS() int64 {
+func (i *fUint16) PTS() int64 {
 	return i.Pts
 }
 
-func (i *FrameUint16) GetNative(x, y int) (Y uint16, Cb uint16, Cr uint16) {
+func (i *fUint16) NextPTS() int64 {
+	return i.NextPts
+}
+
+func (i *fUint16) GetNative(x, y int) (Y uint16, Cb uint16, Cr uint16) {
 	Yindex := x + y*i.properties.Width
 
 	Cwidth := (i.properties.Width * int(i.properties.ColorSpace.ChromaSampling.A)) / int(i.properties.ColorSpace.ChromaSampling.J)
@@ -74,14 +52,57 @@ func (i *FrameUint16) GetNative(x, y int) (Y uint16, Cb uint16, Cr uint16) {
 	return
 }
 
-func (i *FrameUint16) GetNativeLuma() []uint16 {
+func (i *fUint16) FillNativeLuma(buf []uint16) {
+	copy(i.Y, buf)
+}
+
+func (i *fUint16) FillNativeCb(buf []uint16) {
+	copy(i.Cb, buf)
+}
+
+func (i *fUint16) FillNativeCr(buf []uint16) {
+	copy(i.Cr, buf)
+}
+
+func (i *fUint16) GetNativeJoint() []uint16 {
+	// Component slices are allocated as a single buffer
+	return i.Y[:len(i.Y)+len(i.Cb)+len(i.Cr)]
+}
+
+func (i *fUint16) GetJoint() []byte {
+	buf := i.GetNativeJoint()
+	return unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(buf))), len(buf)*2)
+}
+
+func (i *fUint16) GetLuma() []byte {
+	buf := i.GetNativeLuma()
+	return unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(buf))), len(buf)*2)
+}
+
+func (i *fUint16) GetCb() []byte {
+	buf := i.GetNativeCb()
+	return unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(buf))), len(buf)*2)
+}
+
+func (i *fUint16) GetCr() []byte {
+	buf := i.GetNativeCr()
+	return unsafe.Slice((*byte)(unsafe.Pointer(unsafe.SliceData(buf))), len(buf)*2)
+}
+
+func (i *fUint16) GetNativeLuma() []uint16 {
 	return i.Y
 }
 
-func (i *FrameUint16) GetNativeCb() []uint16 {
+func (i *fUint16) GetNativeCb() []uint16 {
 	return i.Cb
 }
 
-func (i *FrameUint16) GetNativeCr() []uint16 {
+func (i *fUint16) GetNativeCr() []uint16 {
 	return i.Cr
 }
+
+func (i *fUint16) Return() {
+	if i.ret != nil {
+		i.ret(i)
+	}
+}

frame/frame_uint8.go

@@ -1,58 +1,38 @@
 package frame
 
-import (
-	"errors"
-)
-
-type FrameUint8 struct {
+type fUint8 struct {
 	properties Properties
+	ret        func(f Frame)
 	Pts        int64
+	NextPts    int64
 	Y          []uint8
 	Cb         []uint8
 	Cr         []uint8
 }
 
-func NewUint8FrameFromBytes(properties Properties, pts int64, data []byte) (*FrameUint8, error) {
-	if frameLength, _ := properties.ColorSpace.FrameSize(properties.Width, properties.Height); frameLength != len(data) {
-		return nil, errors.New("wrong length of data")
-	}
-
-	if properties.ColorSpace.BitDepth > 8 {
-		return nil, errors.New("wrong bit depth")
-	}
-
-	iY := properties.ColorSpace.ChromaSampling.PlaneLumaSamples(properties.Width, properties.Height)
-	iCb := properties.ColorSpace.ChromaSampling.PlaneCbSamples(properties.Width, properties.Height)
-	iCr := properties.ColorSpace.ChromaSampling.PlaneCrSamples(properties.Width, properties.Height)
-
-	return &FrameUint8{
-		properties: properties,
-		Y:          data[:iY],
-		Cb:         data[iY : iY+iCb],
-		Cr:         data[iY+iCb : iY+iCb+iCr],
-		Pts:        pts,
-	}, nil
-}
-
-func (i *FrameUint8) Get16(x, y int) (Y uint16, Cb uint16, Cr uint16) {
+func (i *fUint8) Get16(x, y int) (Y uint16, Cb uint16, Cr uint16) {
 	cy, cb, cr := i.GetNative(x, y)
 
 	return uint16(cy) << (16 - i.properties.ColorSpace.BitDepth), uint16(cb) << (16 - i.properties.ColorSpace.BitDepth), uint16(cr) << (16 - i.properties.ColorSpace.BitDepth)
 }
 
-func (i *FrameUint8) Get8(x, y int) (Y uint8, Cb uint8, Cr uint8) {
+func (i *fUint8) Get8(x, y int) (Y uint8, Cb uint8, Cr uint8) {
 	return i.GetNative(x, y)
 }
 
-func (i *FrameUint8) Properties() Properties {
+func (i *fUint8) Properties() Properties {
 	return i.properties
 }
 
-func (i *FrameUint8) PTS() int64 {
+func (i *fUint8) PTS() int64 {
 	return i.Pts
 }
 
-func (i *FrameUint8) GetNative(x, y int) (Y uint8, Cb uint8, Cr uint8) {
+func (i *fUint8) NextPTS() int64 {
+	return i.NextPts
+}
+
+func (i *fUint8) GetNative(x, y int) (Y uint8, Cb uint8, Cr uint8) {
 	Yindex := x + y*i.properties.Width
 
 	Cwidth := (i.properties.Width * int(i.properties.ColorSpace.ChromaSampling.A)) / int(i.properties.ColorSpace.ChromaSampling.J)
@@ -66,14 +46,53 @@ func (i *FrameUint8) GetNative(x, y int) (Y uint8, Cb uint8, Cr uint8) {
 	return
 }
 
-func (i *FrameUint8) GetNativeLuma() []uint8 {
+func (i *fUint8) FillNativeLuma(buf []uint8) {
+	copy(i.Y, buf)
+}
+
+func (i *fUint8) FillNativeCb(buf []uint8) {
+	copy(i.Cb, buf)
+}
+
+func (i *fUint8) FillNativeCr(buf []uint8) {
+	copy(i.Cr, buf)
+}
+
+func (i *fUint8) GetNativeLuma() []uint8 {
 	return i.Y
 }
 
-func (i *FrameUint8) GetNativeCb() []uint8 {
+func (i *fUint8) GetNativeCb() []uint8 {
 	return i.Cb
 }
 
-func (i *FrameUint8) GetNativeCr() []uint8 {
+func (i *fUint8) GetNativeCr() []uint8 {
 	return i.Cr
 }
+
+func (i *fUint8) GetNativeJoint() []uint8 {
+	// Component slices are allocated as a single buffer
+	return i.Y[:len(i.Y)+len(i.Cb)+len(i.Cr)]
+}
+
+func (i *fUint8) GetJoint() []byte {
+	return i.GetNativeJoint()
+}
+
+func (i *fUint8) GetLuma() []byte {
+	return i.GetNativeLuma()
+}
+
+func (i *fUint8) GetCb() []byte {
+	return i.GetNativeCb()
+}
+
+func (i *fUint8) GetCr() []byte {
+	return i.GetNativeCr()
+}
+
+func (i *fUint8) Return() {
+	if i.ret != nil {
+		i.ret(i)
+	}
+}

frame/pool.go

@@ -0,0 +1,102 @@
+package frame
+
+import (
+	"errors"
+	"sync"
+)
+
+type Pool struct {
+	p          sync.Pool
+	properties Properties
+	frameSize  int
+}
+
+func NewPool(properties Properties) (*Pool, error) {
+	p := &Pool{
+		properties: properties,
+	}
+
+	if properties.ColorSpace.BitDepth > 16 || properties.ColorSpace.BitDepth <= 0 {
+		return nil, errors.New("unsupported bit depth")
+	}
+
+	frameSize, err := properties.ColorSpace.FrameSize(properties.Width, properties.Height)
+	if err != nil {
+		return nil, err
+	}
+
+	p.frameSize = frameSize
+
+	iY := properties.ColorSpace.ChromaSampling.PlaneLumaSamples(properties.Width, properties.Height)
+	iCb := properties.ColorSpace.ChromaSampling.PlaneCbSamples(properties.Width, properties.Height)
+	iCr := properties.ColorSpace.ChromaSampling.PlaneCrSamples(properties.Width, properties.Height)
+
+	// 16-bit frame
+	if properties.ColorSpace.BitDepth > 8 {
+		p.p.New = func() any {
+			buf := make([]uint16, p.frameSize/2)
+			return &fUint16{
+				ret:        p.Put,
+				properties: properties,
+				Y:          buf[:iY],
+				Cb:         buf[iY : iY+iCb],
+				Cr:         buf[iY+iCb : iY+iCb+iCr],
+			}
+		}
+	} else {
+		p.p.New = func() any {
+			buf := make([]uint8, p.frameSize)
+			return &fUint8{
+				ret:        p.Put,
+				properties: properties,
+				Y:          buf[:iY],
+				Cb:         buf[iY : iY+iCb],
+				Cr:         buf[iY+iCb : iY+iCb+iCr],
+			}
+		}
+	}
+
+	return p, nil
+}
+
+func (p *Pool) Properties() Properties {
+	return p.properties
+}
+
+func (p *Pool) Get(pts, nextPts int64) Frame {
+	switch tf := p.p.Get().(type) {
+	case *fUint16:
+		tf.Pts = pts
+		tf.NextPts = nextPts
+		return tf
+	case *fUint8:
+		tf.Pts = pts
+		tf.NextPts = nextPts
+		return tf
+	default:
+		panic("unsupported type")
+	}
+}
+
+func (p *Pool) Put(f Frame) {
+	switch tf := f.(type) {
+	case *fUint16:
+		if tf.properties != p.properties {
+			panic("unsupported properties")
+		}
+		if (len(tf.Y)+len(tf.Cb)+len(tf.Cr))*2 != p.frameSize {
+			panic("unsupported data size")
+		}
+		p.p.Put(tf)
+	case *fUint8:
+		if tf.properties != p.properties {
+			panic("unsupported properties")
+		}
+		if (len(tf.Y) + len(tf.Cb) + len(tf.Cr)) != p.frameSize {
+			panic("unsupported data size")
+		}
+		p.p.Put(tf)
+	default:
+		panic("unsupported type")
+	}
+}

frame/statistics.go

@@ -0,0 +1,85 @@
+package frame
+
+import "math"
+
+type Statistics map[string]any
+
+const (
+	StatisticsKeyFramesIn     = "frames_in"
+	StatisticsKeyFramesOut    = "frames_out"
+	StatisticsKeyLastFrameOut = "last_frame_out"
+)
+
+func (s Statistics) FramesIn() int {
+	if n, ok := s[StatisticsKeyFramesIn].(int); ok {
+		return n
+	}
+	return -1
+}
+
+func (s Statistics) FramesOut() int {
+	if n, ok := s[StatisticsKeyFramesOut].(int); ok {
+		return n
+	}
+	return -1
+}
+
+func (s Statistics) LastFrameOut() *FrameStatistics {
+	if n, ok := s[StatisticsKeyLastFrameOut].(FrameStatistics); ok {
+		return &n
+	}
+	return nil
+}
+
+type FrameStatistics map[string]any
+
+const (
+	FrameStatisticsKeyNumber       = "number"
+	FrameStatisticsKeyPts          = "pts"
+	FrameStatisticsKeyQuantizer    = "quantizer"
+	FrameStatisticsKeySize         = "size"
+	FrameStatisticsKeyIsKeyFrame   = "is_key"
+	FrameStatisticsKeyIsIntraFrame = "is_intra"
+)
+
+func (s FrameStatistics) Number() int {
+	if n, ok := s[FrameStatisticsKeyNumber].(int); ok {
+		return n
+	}
+	return -1
+}
+
+func (s FrameStatistics) Size() int {
+	if n, ok := s[FrameStatisticsKeySize].(int); ok {
+		return n
+	}
+	return -1
+}
+
+func (s FrameStatistics) PTS() int64 {
+	if n, ok := s[FrameStatisticsKeyPts].(int64); ok {
+		return n
+	}
+	return -1
+}
+
+func (s FrameStatistics) Quantizer() float64 {
+	if n, ok := s[FrameStatisticsKeyQuantizer].(float64); ok {
+		return n
+	}
+	return math.NaN()
+}
+
+func (s FrameStatistics) IsKeyFrame() bool {
+	if n, ok := s[FrameStatisticsKeyIsKeyFrame].(bool); ok {
+		return n
+	}
+	return false
+}
+
+func (s FrameStatistics) IsIntraFrame() bool {
+	if n, ok := s[FrameStatisticsKeyIsIntraFrame].(bool); ok {
+		return n
+	}
+	return false
+}

frame/stream.go

@@ -226,17 +226,19 @@ func (s *Stream) Monochrome() *Stream {
 
 			switch typedFrame := f.(type) {
 			case TypedFrame[uint8]:
-				channel <- &FrameUint8{
+				channel <- &fUint8{
 					properties: frameProps,
 					Pts:        typedFrame.PTS(),
+					NextPts:    typedFrame.NextPTS(),
 					Y:          typedFrame.GetNativeLuma(),
 					Cb:         nil,
 					Cr:         nil,
 				}
 			case TypedFrame[uint16]:
-				channel <- &FrameUint16{
+				channel <- &fUint16{
 					properties: frameProps,
 					Pts:        typedFrame.PTS(),
+					NextPts:    typedFrame.NextPTS(),
 					Y:          typedFrame.GetNativeLuma(),
 					Cb:         nil,
 					Cr:         nil,

testdata/testdata.go

@@ -2,12 +2,13 @@ package testdata
 
 import (
 	"git.gammaspectra.live/S.O.N.G/Ignite/color"
+	"git.gammaspectra.live/S.O.N.G/Ignite/utilities"
 	"io"
 	"net/http"
 	"os"
 	"path"
 	"runtime"
-	"testing"
+	"time"
 )
 
 func init() {
@@ -26,11 +27,16 @@ type TestSample struct {
 	Type          string
 	Width, Height int
 	Frames        int
+	TimeBase      utilities.Ratio
 	ColorSpace    color.Space
 	SkipNotFound  bool
 }
 
-func (sample *TestSample) Open(t *testing.T) (io.ReadCloser, error) {
+func (sample *TestSample) Duration() time.Duration {
+	return sample.TimeBase.PTSToDuration(int64(sample.Frames - 1))
+}
+
+func (sample *TestSample) Open(t TestRunner) (io.ReadCloser, error) {
 	var reader io.ReadCloser
 	var err error
 
@@ -54,6 +60,11 @@ func (sample *TestSample) Open(t *testing.T) (io.ReadCloser, error) {
 	return reader, nil
 }
 
+type TestRunner interface {
+	Skip(args ...any)
+	Fatal(args ...any)
+}
+
 var (
 	Y4M_Sintel_Trailer_720p24_YUV420_8bit = TestSample{
 		Path: "testdata/sintel_trailer_2k_720p24.y4m",
@@ -63,6 +74,7 @@ var (
 		Width:      1280,
 		Height:     720,
 		Frames:     1253,
+		TimeBase:   utilities.NewRatio(1, 24),
 		ColorSpace: color.MustColorFormatFromString("420jpeg"),
 	}
 	Y4M_Big_Buck_Bunny_360p24_YUV420_8bit = TestSample{
@@ -72,6 +84,7 @@ var (
 		Width:        640,
 		Height:       360,
 		Frames:       14315,
+		TimeBase:     utilities.NewRatio(1, 24),
 		ColorSpace:   color.MustColorFormatFromString("420jpeg"),
 		SkipNotFound: true,
 	}
@@ -83,6 +96,7 @@ var (
 		Width:      1280,
 		Height:     720,
 		Frames:     500,
+		TimeBase:   utilities.NewRatio(1, 50),
 		ColorSpace: color.MustColorFormatFromString("444p8"),
 	}
 	Y4M_Ducks_Take_Off_720p50_YUV422_8bit = TestSample{
@@ -93,6 +107,7 @@ var (
 		Width:      1280,
 		Height:     720,
 		Frames:     500,
+		TimeBase:   utilities.NewRatio(1, 50),
 		ColorSpace: color.MustColorFormatFromString("422p8"),
 	}
 	Y4M_Netflix_FoodMarket_2160p60_YUV420_10bit = TestSample{
@@ -102,6 +117,7 @@ var (
 		Width:        4096,
 		Height:       2160,
 		Frames:       600,
+		TimeBase:     utilities.NewRatio(1, 60),
 		ColorSpace:   color.MustColorFormatFromString("420p10"),
 		SkipNotFound: true,
 	}
@@ -113,6 +129,7 @@ var (
 		Width:      1280,
 		Height:     720,
 		Frames:     1253,
+		TimeBase:   utilities.NewRatio(1, 24),
 		ColorSpace: color.MustColorFormatFromString("420jpeg"),
 	}
 	AV1_Netflix_Sol_Levante_2160p24_YUV444_12bit_Lossy = TestSample{
@@ -122,6 +139,7 @@ var (
 		Width:      3840,
 		Height:     2160,
 		Frames:     6313,
+		TimeBase:   utilities.NewRatio(1, 24),
 		ColorSpace: color.MustColorFormatFromString("444p12"),
 	}
 )

utilities/libvmaf/libvmaf.go

@@ -121,26 +121,12 @@ func (v *VMAF) frameToPicture(f frame.Frame) *C.VmafPicture {
 	if p := v.allocatePicture(f.Properties()); p == nil {
 		return nil
 	} else {
-		//TODO: check validity of lengths
-		if f16, ok := f.(frame.TypedFrame[uint16]); ok {
-			yPlane := unsafe.Slice((*uint16)(p.data[0]), len(f16.GetNativeLuma()))
-			uPlane := unsafe.Slice((*uint16)(p.data[1]), len(f16.GetNativeCb()))
-			vPlane := unsafe.Slice((*uint16)(p.data[2]), len(f16.GetNativeCr()))
-			copy(yPlane, f16.GetNativeLuma())
-			copy(uPlane, f16.GetNativeCb())
-			copy(vPlane, f16.GetNativeCr())
-		} else if f8, ok := f.(frame.TypedFrame[uint8]); ok {
-			yPlane := unsafe.Slice((*uint8)(p.data[0]), len(f8.GetNativeLuma()))
-			uPlane := unsafe.Slice((*uint8)(p.data[1]), len(f8.GetNativeCb()))
-			vPlane := unsafe.Slice((*uint8)(p.data[2]), len(f8.GetNativeCr()))
-			copy(yPlane, f8.GetNativeLuma())
-			copy(uPlane, f8.GetNativeCb())
-			copy(vPlane, f8.GetNativeCr())
-		} else {
-			// not supported frame
-			v.deallocatePicture(p)
-			return nil
-		}
+		luma := f.GetLuma()
+		cb := f.GetCb()
+		cr := f.GetCr()
+		copy(unsafe.Slice((*byte)(p.data[0]), len(luma)), luma)
+		copy(unsafe.Slice((*byte)(p.data[1]), len(cb)), cb)
+		copy(unsafe.Slice((*byte)(p.data[2]), len(cr)), cr)
 
 		return p
 	}

utilities/ratio.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"gopkg.in/yaml.v3"
 	"math"
+	"time"
 )
 
 type Ratio struct {
@@ -15,6 +16,10 @@ func (r Ratio) Float64() float64 {
 	return float64(r.Numerator) / float64(r.Denominator)
 }
 
+func (r Ratio) PTSToDuration(pts int64) time.Duration {
+	return (time.Duration(pts) * time.Second * time.Duration(r.Numerator)) / time.Duration(r.Denominator)
+}
+
 func (r *Ratio) UnmarshalJSON(buf []byte) error {
 	_, err := fmt.Sscanf(string(buf), "\"%d:%d\"", &r.Numerator, &r.Denominator)
 	return err