How to send my video captured by Gocv to the Browser with WebRTC

[maydersonmellops]

I want to capture my video using Gocv and send it to the Browser using WebRTC with VP8, similar to what was done in the example using mediadevices but I need it to be with Gocv, I tried using the code below but without success, there is no error but the video is there in the Browser does not play, I tested with other browsers but without success.

Your environment.

• Version: Mac OS M1 Pro - Ventura
• Browser: Google Chrome 118.0.5993.117 (Official Build) (arm64)
• Golang: 1.21

What did you do?

Backend

package main

import (
	"fmt"
	"time"

	signal "github.com/Sup3r-Us3r/test-app/internal"
	"github.com/pion/mediadevices"
	"github.com/pion/mediadevices/pkg/codec/vpx"
	"github.com/pion/webrtc/v3"
	"github.com/pion/webrtc/v3/pkg/media"
	"gocv.io/x/gocv"
)

func main() {
	webcam, _ := gocv.OpenVideoCapture(0)
	defer webcam.Close()

	webRTCConfig := webrtc.Configuration{
		ICEServers: []webrtc.ICEServer{
			{
				URLs: []string{"stun:stun.l.google.com:19302"},
			},
		},
	}

	// Wait for the offer to be pasted
	offer := webrtc.SessionDescription{}
	signal.Decode(signal.MustReadStdin(), &offer)

	// Create a new RTCPeerConnection
	vp8Params, err := vpx.NewVP8Params()
	if err != nil {
		panic(err)
	}
	vp8Params.BitRate = 500_000 // 500kbps

	codecSelector := mediadevices.NewCodecSelector(
		mediadevices.WithVideoEncoders(&vp8Params),
	)

	// Configure WebRTC Connection
	mediaEngine := webrtc.MediaEngine{}
	mediaEngine.RegisterDefaultCodecs()
	codecSelector.Populate(&mediaEngine)
	api := webrtc.NewAPI(webrtc.WithMediaEngine(&mediaEngine))
	peerConnection, _ := api.NewPeerConnection(webRTCConfig)

	// Set the handler for ICE connection state
	// This will notify you when the peer has connected/disconnected
	peerConnection.OnICEConnectionStateChange(func(connectionState webrtc.ICEConnectionState) {
		fmt.Printf("Connection State has changed %s \n", connectionState.String())
	})

	// Create track video
	videoTrack, _ := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8}, "video", "pion")

	// Add track to PeerConnection
	_, err = peerConnection.AddTrack(videoTrack)
	if err != nil {
		fmt.Println("\nERROR ADD TRACK: ", err)
	}

	// Set the remote SessionDescription
	err = peerConnection.SetRemoteDescription(offer)
	if err != nil {
		panic(err)
	}

	// Create an answer
	answer, err := peerConnection.CreateAnswer(nil)
	if err != nil {
		panic(err)
	}

	// Create channel that is blocked until ICE Gathering is complete
	gatherComplete := webrtc.GatheringCompletePromise(peerConnection)

	// Sets the LocalDescription, and starts our UDP listeners
	err = peerConnection.SetLocalDescription(answer)
	if err != nil {
		panic(err)
	}

	// Block until ICE Gathering is complete, disabling trickle ICE
	// we do this because we only can exchange one signaling message
	// in a production application you should exchange ICE Candidates via OnICECandidate
	<-gatherComplete

	// Output the answer in base64 so we can paste it in browser
	fmt.Println(signal.Encode(*peerConnection.LocalDescription()))

	img := gocv.NewMat()
	defer img.Close()


	ticker := time.NewTicker(time.Second * 3)
	defer ticker.Stop()

	for {
		if ok := webcam.Read(&img); !ok {
			fmt.Printf("Device closed: %v\n", 0)
			return
		}

		if img.Empty() {
			continue
		}

		// Encode the frame in VP8
		frame, _ := gocv.IMEncode(gocv.JPEGFileExt, img)
		err = videoTrack.WriteSample(media.Sample{Data: frame.GetBytes(), Duration: time.Second})
		if err != nil {
			fmt.Println("\nERROR: ", err)
		}


		// select {
		// case <-ticker.C:
		// 	table := crc32.MakeTable(crc32.IEEE)
		// 	checksum := crc32.Checksum([]byte(videoTrack.StreamID()), table)

		// 	if rtcpSendErr := peerConnection.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: checksum}}); rtcpSendErr != nil {
		// 		fmt.Println(rtcpSendErr)
		// 	}
		// default:
		// }
	}
}

Client

const pc = new RTCPeerConnection({
  iceServers: [
    {
      urls: 'stun:stun.l.google.com:19302',
    },
  ],
});
const log = (msg) => {
  document.getElementById('logs').innerHTML += msg + '<br>';
};

pc.ontrack = (event) => {
  const el = document.createElement(event.track.kind);
  el.srcObject = event.streams[0];
  el.autoplay = true;
  el.controls = true;
  document.getElementById('remoteVideos').appendChild(el);
};

pc.oniceconnectionstatechange = (e) => log(pc.iceConnectionState);
pc.onicecandidate = (event) => {
  if (event.candidate === null) {
    document.getElementById('localSessionDescription').value = btoa(
      JSON.stringify(pc.localDescription)
    );
  }
};

// Offer to receive 1 audio, and 1 video tracks
pc.addTransceiver('video', {
  direction: 'recvonly',
});
pc.createOffer()
  .then((d) => pc.setLocalDescription(d))
  .catch(log);

window.startSession = () => {
  const sd = document.getElementById('remoteSessionDescription').value;
  if (sd === '') {
    return alert('Session Description must not be empty');
  }
  try {
    pc.setRemoteDescription(new RTCSessionDescription(JSON.parse(atob(sd))));
  } catch (e) {
    alert(e);
  }
};

[Sean-Der]

Hi @maydersonmellops. Sorry I didn't answer this month backs. Just getting around to finally triaging things.

Looking at the code my assumption was that you weren't properly encoding to VP8. frame, _ := gocv.IMEncode(gocv.JPEGFileExt, img) is taking it to jpeg?

Hopefully you figured it out! If you are still struggling I would love to help though!

[qidailove]

May I ask if you have solved this problem? I have also encountered this problem now. I have established a connection and the front-end web also shows that the stream has been received, but the video has not been played

[maydersonmellops]

Hello @qidailove I couldn't solve this either, I still have the same problem, I tried to encode to VP8 but without success, @Sean-Der do you have any example for this process?

[qidailove]

Hello, I have also tried using ffmpeg to convert JPEG image data into VP8 format encoded data, but the front-end page still does not display it

[maydersonmellops]

Got it, let's see if @Sean-Der can support us with this.

[qidailove]

yeah