diff --git a/pkg/sfu/downtrack.go b/pkg/sfu/downtrack.go
index 2a1fa0e21..05d8b6373 100644
--- a/pkg/sfu/downtrack.go
+++ b/pkg/sfu/downtrack.go
@@ -397,6 +397,9 @@ func (d *DownTrack) writeSimulcastRTP(extPkt *buffer.ExtPacket) error {
 	newSN := extPkt.Packet.SequenceNumber - d.snOffset
 	newTS := extPkt.Packet.Timestamp - d.tsOffset
 
+	d.payload = d.payload[:len(extPkt.Packet.Payload)]
+	copy(d.payload, extPkt.Packet.Payload)
+
 	var (
 		picID   uint16
 		tlz0Idx uint8
@@ -404,7 +407,7 @@ func (d *DownTrack) writeSimulcastRTP(extPkt *buffer.ExtPacket) error {
 	if d.simulcast.temporalSupported {
 		if d.mime == "video/vp8" {
 			drop := false
-			if extPkt.Packet.Payload, picID, tlz0Idx, drop = setVP8TemporalLayer(extPkt, d); drop {
+			if picID, tlz0Idx, drop = setVP8TemporalLayer(extPkt, d); drop {
 				// Pkt not in temporal getLayer update sequence number offset to avoid gaps
 				d.snOffset++
 				return nil
@@ -438,7 +441,7 @@ func (d *DownTrack) writeSimulcastRTP(extPkt *buffer.ExtPacket) error {
 	hdr.SSRC = d.ssrc
 	hdr.PayloadType = d.payloadType
 
-	_, err := d.writeStream.WriteRTP(&hdr, extPkt.Packet.Payload)
+	_, err := d.writeStream.WriteRTP(&hdr, d.payload)
 	if err != nil {
 		Logger.Error(err, "Write packet err")
 	}
diff --git a/pkg/sfu/helpers.go b/pkg/sfu/helpers.go
index b314780d8..70ec97058 100644
--- a/pkg/sfu/helpers.go
+++ b/pkg/sfu/helpers.go
@@ -30,10 +30,10 @@ func (a *atomicBool) get() bool {
 // setVp8TemporalLayer is a helper to detect and modify accordingly the vp8 payload to reflect
 // temporal changes in the SFU.
 // VP8 temporal layers implemented according https://tools.ietf.org/html/rfc7741
-func setVP8TemporalLayer(p *buffer.ExtPacket, s *DownTrack) (payload []byte, picID uint16, tlz0Idx uint8, drop bool) {
+func setVP8TemporalLayer(p *buffer.ExtPacket, s *DownTrack) (picID uint16, tlz0Idx uint8, drop bool) {
 	pkt, ok := p.Payload.(buffer.VP8)
 	if !ok {
-		return p.Packet.Payload, 0, 0, false
+		return 0, 0, false
 	}
 
 	layer := atomic.LoadInt32(&s.temporalLayer)
@@ -48,10 +48,6 @@ func setVP8TemporalLayer(p *buffer.ExtPacket, s *DownTrack) (payload []byte, pic
 		drop = true
 		return
 	}
-	// If we are here modify payload
-	payload = s.payload
-	payload = payload[:len(p.Packet.Payload)]
-	copy(payload, p.Packet.Payload)
 
 	picID = pkt.PictureID - s.simulcast.refPicID + s.simulcast.pRefPicID + 1
 	tlz0Idx = pkt.TL0PICIDX - s.simulcast.refTlZIdx + s.simulcast.pRefTlZIdx + 1
@@ -61,7 +57,7 @@ func setVP8TemporalLayer(p *buffer.ExtPacket, s *DownTrack) (payload []byte, pic
 		s.simulcast.lTlZIdx = tlz0Idx
 	}
 
-	modifyVP8TemporalPayload(payload, pkt.PicIDIdx, pkt.TlzIdx, picID, tlz0Idx, pkt.MBit)
+	modifyVP8TemporalPayload(s.payload, pkt.PicIDIdx, pkt.TlzIdx, picID, tlz0Idx, pkt.MBit)
 
 	return
 }
@@ -75,7 +71,6 @@ func modifyVP8TemporalPayload(payload []byte, picIDIdx, tlz0Idx int, picID uint1
 		payload[picIDIdx+1] = pid[1]
 	}
 	payload[tlz0Idx] = tlz0ID
-
 }
 
 func timeToNtp(ns int64) uint64 {