Improve screen mirroring performance and add stream encoding options

ext/src/cast/senders/mirroring.ts

@@ -30,7 +30,14 @@ class MirroringSender {
     private session?: Session;
     private wasSessionRequested = false;
 
-    private peerConnection?: RTCPeerConnection;
+    private peerConnection: Optional<RTCPeerConnection>;
+
+    // Stream opts
+    private streamMaxFrameRate = 1;
+    private streamMaxBitRate = 1;
+    private streamDownscaleFactor = 1;
+    private streamUseMaxResolution = false;
+    private streamMaxResolution: { width?: number; height?: number } = {};
 
     constructor(opts: MirroringSenderOpts) {
         this.contextTabId = opts.contextTabId;
@@ -49,7 +56,21 @@ class MirroringSender {
             logger.error("Failed to initialize cast API", err);
         }
 
-        const mirroringAppId = await options.get("mirroringAppId");
+        const {
+            mirroringAppId,
+            mirroringStreamMaxFrameRate,
+            mirroringStreamMaxBitRate,
+            mirroringStreamDownscaleFactor,
+            mirroringStreamUseMaxResolution,
+            mirroringStreamMaxResolution
+        } = await options.getAll();
+
+        this.streamMaxFrameRate = mirroringStreamMaxFrameRate;
+        this.streamMaxBitRate = mirroringStreamMaxBitRate;
+        this.streamDownscaleFactor = mirroringStreamDownscaleFactor;
+        this.streamUseMaxResolution = mirroringStreamUseMaxResolution;
+        this.streamMaxResolution = mirroringStreamMaxResolution;
+
         const sessionRequest = new cast.SessionRequest(mirroringAppId);
 
         const apiConfig = new cast.ApiConfig(
@@ -61,6 +82,11 @@ class MirroringSender {
         cast.initialize(apiConfig);
     }
 
+    stop() {
+        this.peerConnection?.close();
+        this.session?.stop();
+    }
+
     private sessionListener() {
         // Unused
     }
@@ -87,27 +113,24 @@ class MirroringSender {
     }
 
     private async createMirroringConnection() {
+        const pc = new RTCPeerConnection();
+        this.peerConnection = pc;
+
         this.session?.addMessageListener(NS_FX_CAST, async (_ns, message) => {
             const parsedMessage = JSON.parse(message) as MirroringAppMessage;
             switch (parsedMessage.subject) {
                 case "peerConnectionAnswer":
-                    this.peerConnection?.setRemoteDescription(
-                        parsedMessage.data
-                    );
+                    pc.setRemoteDescription(parsedMessage.data);
                     break;
                 case "iceCandidate":
-                    this.peerConnection?.addIceCandidate(parsedMessage.data);
+                    pc.addIceCandidate(parsedMessage.data);
                     break;
             }
         });
 
-        this.peerConnection = new RTCPeerConnection();
-
-        this.peerConnection.addEventListener("negotiationneeded", async () => {
-            if (!this.peerConnection) return;
-
-            const offer = await this.peerConnection.createOffer();
-            await this.peerConnection.setLocalDescription(offer);
+        pc.addEventListener("negotiationneeded", async () => {
+            const offer = await pc.createOffer();
+            await pc.setLocalDescription(offer);
 
             this.sendMirroringAppMessage({
                 subject: "peerConnectionOffer",
@@ -115,7 +138,7 @@ class MirroringSender {
             });
         });
 
-        this.peerConnection.addEventListener("icecandidate", ev => {
+        pc.addEventListener("icecandidate", ev => {
             if (!ev.candidate) return;
             this.sendMirroringAppMessage({
                 subject: "iceCandidate",
@@ -123,19 +146,89 @@ class MirroringSender {
             });
         });
 
+        // Connection listener
+        pc.addEventListener("iceconnectionstatechange", async () => {
+            if (pc.iceConnectionState !== "connected") {
+                return;
+            }
+
+            applyParameters();
+        });
+
+        /** Applies stream encoding parameters.  */
+        const applyParameters = async () => {
+            // Set stream encoding parameters
+            const [sender] = pc.getSenders();
+            const params = sender.getParameters();
+            if (!params.encodings) {
+                params.encodings = [{}];
+            }
+
+            const [encoding] = params.encodings;
+
+            if (!(encoding as any).maxFramerate) {
+                (encoding as any).maxFramerate = this.streamMaxFrameRate;
+            }
+            if (!encoding.maxBitrate) {
+                encoding.maxBitrate = this.streamMaxBitRate;
+            }
+
+            encoding.scaleResolutionDownBy = this.streamDownscaleFactor;
+
+            // Handle limiting stream resolution
+            if (this.streamUseMaxResolution) {
+                const { width: trackWidth, height: trackHeight } =
+                    sender.track?.getSettings() ?? {};
+
+                // Calculate downscale ratios for width/height
+                let widthRatio = 1;
+                let heightRatio = 1;
+                if (trackWidth && this.streamMaxResolution.width) {
+                    widthRatio = trackWidth / this.streamMaxResolution.width;
+                }
+                if (trackHeight && this.streamMaxResolution.height) {
+                    heightRatio = trackHeight / this.streamMaxResolution.height;
+                }
+
+                // Use the largest ratio to ensure below resolution limit
+                const downscaleRatio = Math.max(1, widthRatio, heightRatio);
+
+                // Multiply existing downscale
+                encoding.scaleResolutionDownBy *= downscaleRatio;
+            }
+
+            await sender.setParameters(params);
+        };
+
+        let stream: MediaStream;
         try {
             // Add screen media stream
-            this.peerConnection.addStream(
-                await navigator.mediaDevices.getDisplayMedia({
-                    video: { cursor: "motion" },
-                    audio: false
-                })
-            );
+
+            stream = await navigator.mediaDevices.getDisplayMedia({
+                video: {
+                    cursor: "motion",
+                    frameRate: this.streamMaxFrameRate
+                },
+                audio: false
+            });
+
+            const [track] = stream.getVideoTracks();
+            pc.addTrack(track, stream);
+            track.addEventListener("ended", () => this.stop());
         } catch (err) {
             logger.error("Failed to add display media stream!", err);
-            this.peerConnection.close();
-            this.session?.stop();
+            this.stop();
+            return;
         }
+
+        /**
+         * Use a video element to get stream resize events and update
+         * scaling parameters.
+         */
+        const video = document.createElement("video");
+        video.srcObject = stream;
+        video.addEventListener("resize", () => applyParameters());
+        video.play();
     }
 }
 
@@ -145,8 +238,12 @@ class MirroringSender {
 if (window.location.protocol !== "moz-extension:") {
     const window_ = window as any;
 
-    new MirroringSender({
+    const sender = new MirroringSender({
         contextTabId: window_.contextTabId,
         receiverDevice: window_.receiverDevice
     });
+
+    window.addEventListener("beforeunload", () => {
+        sender.stop();
+    });
 }