feat: create capture subscription api to support more transports (#699)

* feat: introduce encoded media subscriptions

* chore: preserve sample field comments

* perf: reduce encoded sample dispatch copies

* docs: define encoded sample buffer ownership

* fix: avoid pipeline recreation errors

Reuse pipelines started by a new subscription during resize.

* fix: synchronize stream bitrate state

Protect bitrate updates, reads, and resets with one mutex.

* simplify

* group stream sink fields

* fix: remove unnecessary blank line in removeSubscription method

* remove defensive check

* remove noop

---------

Co-authored-by: Miroslav Šedivý <sedivy.miro@gmail.com>

server/pkg/gst/gst.c

@@ -87,18 +87,19 @@ static GstFlowReturn gstreamer_send_new_sample_handler(GstElement *object, gpoin
   GstPipelineCtx *ctx = (GstPipelineCtx *)user_data;
   GstSample *sample = NULL;
   GstBuffer *buffer = NULL;
-  gpointer copy = NULL;
-  gsize copy_size = 0;
+  GstMapInfo map = {0};
 
   g_signal_emit_by_name(object, "pull-sample", &sample);
   if (sample) {
     buffer = gst_sample_get_buffer(sample);
-    if (buffer) {
-      gst_buffer_extract_dup(buffer, 0, gst_buffer_get_size(buffer), &copy, &copy_size);
-      goHandlePipelineBuffer(ctx->pipelineId, copy, copy_size,
+    if (buffer && gst_buffer_map(buffer, &map, GST_MAP_READ)) {
+      goHandlePipelineBuffer(ctx->pipelineId, map.data, (int)map.size,
+        GST_BUFFER_PTS(buffer),
+        GST_BUFFER_DTS(buffer),
         GST_BUFFER_DURATION(buffer),
         GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT)
       );
+      gst_buffer_unmap(buffer, &map);
     }
     gst_sample_unref(sample);
   }

server/pkg/gst/gst.go

@@ -211,9 +211,7 @@ func CheckElement(element string) error {
 }
 
 //export goHandlePipelineBuffer
-func goHandlePipelineBuffer(pipelineID C.int, buf C.gpointer, bufLen C.int, duration C.guint64, deltaUnit C.gboolean) {
-	defer C.g_free(buf)
-
+func goHandlePipelineBuffer(pipelineID C.int, buf C.gpointer, bufLen C.int, pts C.guint64, dts C.guint64, duration C.guint64, deltaUnit C.gboolean) {
 	pipelinesLock.Lock()
 	pipeline, ok := pipelines[int(pipelineID)]
 	pipelinesLock.Unlock()
@@ -223,6 +221,8 @@ func goHandlePipelineBuffer(pipelineID C.int, buf C.gpointer, bufLen C.int, dura
 			Data:      C.GoBytes(unsafe.Pointer(buf), bufLen),
 			Length:    int(bufLen),
 			Timestamp: time.Now(),
+			PTS:       time.Duration(pts),
+			DTS:       time.Duration(dts),
 			Duration:  time.Duration(duration),
 			DeltaUnit: deltaUnit == C.TRUE,
 		}

server/pkg/gst/gst.h

@@ -23,7 +23,7 @@ typedef struct GstPipelineCtx {
   GstElement *appsrc;
 } GstPipelineCtx;
 
-extern void goHandlePipelineBuffer(int pipelineId, void *buffer, int bufferLen, guint64 duration, gboolean deltaUnit);
+extern void goHandlePipelineBuffer(int pipelineId, void *buffer, int bufferLen, guint64 pts, guint64 dts, guint64 duration, gboolean deltaUnit);
 extern void goPipelineLog(int pipelineId, char *level, char *msg);
 
 GstPipelineCtx *gstreamer_pipeline_create(char *pipelineStr, int pipelineId, GError **error);

server/pkg/types/capture.go

@@ -20,19 +20,37 @@ var (
 type Sample struct {
 	// timing information
 	Timestamp time.Time
-	Duration  time.Duration
+	// PTS, DTS, and Duration use -1 for GST_CLOCK_TIME_NONE.
+	PTS      time.Duration
+	DTS      time.Duration
+	Duration time.Duration
 	// metadata
 	DeltaUnit bool // this unit cannot be decoded independently.
 	// buffer length
 	Length int
-	// buffer with encoded media
+	// Data is immutable and remains valid while referenced. Consumers may retain
+	// it after WriteSample returns.
 	Data []byte
 }
 
-type SampleListener interface {
+type SampleConsumer interface {
 	WriteSample(Sample)
 }
 
+type EncodedStream interface {
+	ID() string
+	Codec() codec.RTPCodec
+	Bitrate() uint64
+
+	Subscribe(SampleConsumer) (StreamSubscription, error)
+}
+
+type StreamSubscription interface {
+	Stream() EncodedStream
+	Switch(EncodedStream) error
+	Close() error
+}
+
 type BroadcastManager interface {
 	Start(url string) error
 	Stop()
@@ -103,27 +121,16 @@ type StreamSelector struct {
 	Bitrate uint64 `json:"bitrate"`
 }
 
-type StreamSelectorManager interface {
+type EncodedStreamSelector interface {
 	IDs() []string
 	Codec() codec.RTPCodec
 
-	GetStream(selector StreamSelector) (StreamSinkManager, bool)
+	GetStream(selector StreamSelector) (EncodedStream, bool)
 }
 
-type StreamSinkManager interface {
-	ID() string
-	Codec() codec.RTPCodec
-	Bitrate() uint64
-
-	AddListener(listener SampleListener) error
-	RemoveListener(listener SampleListener) error
-	MoveListenerTo(listener SampleListener, targetStream StreamSinkManager) error
-
-	ListenersCount() int
-	Started() bool
-
-	CreatePipeline() error
-	DestroyPipeline()
+type EncodedMediaSource interface {
+	Audio() EncodedStream
+	Video() EncodedStreamSelector
 }
 
 type StreamSrcManager interface {
@@ -137,13 +144,13 @@ type StreamSrcManager interface {
 }
 
 type CaptureManager interface {
+	EncodedMediaSource
+
 	Start()
 	Shutdown() error
 
 	Broadcast() BroadcastManager
 	Screencast() ScreencastManager
-	Audio() StreamSinkManager
-	Video() StreamSelectorManager
 
 	Webcam() StreamSrcManager
 	Microphone() StreamSrcManager