Improve screen mirroring performance and add stream encoding options

ext/src/_locales/en/messages.json

@@ -498,6 +498,46 @@
         "message": "App ID for a registered Chromecast receiver application. Advanced use only. Must be compatible with the default app (see GitHub repo).",
         "description": "Mirroring app ID option description."
     },
+    "optionsMirroringMediaStreamOptions": {
+        "message": "Media stream encoding options",
+        "description": "Options page mirroring category description."
+    },
+    "optionsMirroringStreamFrameRate": {
+        "message": "Max frame rate:",
+        "description": "Mirroring stream max frame rate option label."
+    },
+    "optionsMirroringStreamMaxBitRate": {
+        "message": "Max bitrate:",
+        "description": "Mirroring stream max bit rate option label."
+    },
+    "optionsMirroringStreamMaxBitRateDescription": {
+        "message": "Maximum bitrate in bits per second.",
+        "description": "Mirroring stream max bit rate option description."
+    },
+    "optionsMirroringStreamDownscaleFactor": {
+        "message": "Downscale factor:",
+        "description": "Mirroring stream downscale factor option label."
+    },
+    "optionsMirroringStreamDownscaleFactorDescription": {
+        "message": "Factor by which to scale down the video stream e.g. a factor of 2.0 would result in a video 1/4 the size. ",
+        "description": "Mirroring stream downscale factor option description."
+    },
+    "optionsMirroringStreamMaxResolution": {
+        "message": "Limit resolution to",
+        "description": "Mirroring stream resolution option label."
+    },
+    "optionsMirroringStreamMaxResolutionDescription": {
+        "message": "Limits the maximum video stream resolution whilst maintaining the source aspect ratio.",
+        "description": "Mirroring stream resolution option description."
+    },
+    "optionsMirroringStreamMaxResolutionWidthPlaceholder": {
+        "message": "Width",
+        "description": "Max resolution width input placeholder."
+    },
+    "optionsMirroringStreamMaxResolutionHeightPlaceholder": {
+        "message": "Height",
+        "description": "Max resolution height input placeholder."
+    },
 
     "optionsOptionRecommended": {
         "message": "recommended",

ext/src/cast/senders/mirroring.ts

@@ -30,7 +30,14 @@ class MirroringSender {
     private session?: Session;
     private wasSessionRequested = false;
 
-    private peerConnection?: RTCPeerConnection;
+    private peerConnection: Optional<RTCPeerConnection>;
+
+    // Stream opts
+    private streamMaxFrameRate = 1;
+    private streamMaxBitRate = 1;
+    private streamDownscaleFactor = 1;
+    private streamUseMaxResolution = false;
+    private streamMaxResolution: { width?: number; height?: number } = {};
 
     constructor(opts: MirroringSenderOpts) {
         this.contextTabId = opts.contextTabId;
@@ -49,7 +56,21 @@ class MirroringSender {
             logger.error("Failed to initialize cast API", err);
         }
 
-        const mirroringAppId = await options.get("mirroringAppId");
+        const {
+            mirroringAppId,
+            mirroringStreamMaxFrameRate,
+            mirroringStreamMaxBitRate,
+            mirroringStreamDownscaleFactor,
+            mirroringStreamUseMaxResolution,
+            mirroringStreamMaxResolution
+        } = await options.getAll();
+
+        this.streamMaxFrameRate = mirroringStreamMaxFrameRate;
+        this.streamMaxBitRate = mirroringStreamMaxBitRate;
+        this.streamDownscaleFactor = mirroringStreamDownscaleFactor;
+        this.streamUseMaxResolution = mirroringStreamUseMaxResolution;
+        this.streamMaxResolution = mirroringStreamMaxResolution;
+
         const sessionRequest = new cast.SessionRequest(mirroringAppId);
 
         const apiConfig = new cast.ApiConfig(
@@ -61,6 +82,11 @@ class MirroringSender {
         cast.initialize(apiConfig);
     }
 
+    stop() {
+        this.peerConnection?.close();
+        this.session?.stop();
+    }
+
     private sessionListener() {
         // Unused
     }
@@ -87,27 +113,24 @@ class MirroringSender {
     }
 
     private async createMirroringConnection() {
+        const pc = new RTCPeerConnection();
+        this.peerConnection = pc;
+
         this.session?.addMessageListener(NS_FX_CAST, async (_ns, message) => {
             const parsedMessage = JSON.parse(message) as MirroringAppMessage;
             switch (parsedMessage.subject) {
                 case "peerConnectionAnswer":
-                    this.peerConnection?.setRemoteDescription(
+                    pc.setRemoteDescription(parsedMessage.data);
-                        parsedMessage.data
-                    );
                     break;
                 case "iceCandidate":
-                    this.peerConnection?.addIceCandidate(parsedMessage.data);
+                    pc.addIceCandidate(parsedMessage.data);
                     break;
             }
         });
 
-        this.peerConnection = new RTCPeerConnection();
+        pc.addEventListener("negotiationneeded", async () => {
-
+            const offer = await pc.createOffer();
-        this.peerConnection.addEventListener("negotiationneeded", async () => {
+            await pc.setLocalDescription(offer);
-            if (!this.peerConnection) return;
-
-            const offer = await this.peerConnection.createOffer();
-            await this.peerConnection.setLocalDescription(offer);
 
             this.sendMirroringAppMessage({
                 subject: "peerConnectionOffer",
@@ -115,7 +138,7 @@ class MirroringSender {
             });
         });
 
-        this.peerConnection.addEventListener("icecandidate", ev => {
+        pc.addEventListener("icecandidate", ev => {
             if (!ev.candidate) return;
             this.sendMirroringAppMessage({
                 subject: "iceCandidate",
@@ -123,19 +146,89 @@ class MirroringSender {
             });
         });
 
+        // Connection listener
+        pc.addEventListener("iceconnectionstatechange", async () => {
+            if (pc.iceConnectionState !== "connected") {
+                return;
+            }
+
+            applyParameters();
+        });
+
+        /** Applies stream encoding parameters.  */
+        const applyParameters = async () => {
+            // Set stream encoding parameters
+            const [sender] = pc.getSenders();
+            const params = sender.getParameters();
+            if (!params.encodings) {
+                params.encodings = [{}];
+            }
+
+            const [encoding] = params.encodings;
+
+            if (!(encoding as any).maxFramerate) {
+                (encoding as any).maxFramerate = this.streamMaxFrameRate;
+            }
+            if (!encoding.maxBitrate) {
+                encoding.maxBitrate = this.streamMaxBitRate;
+            }
+
+            encoding.scaleResolutionDownBy = this.streamDownscaleFactor;
+
+            // Handle limiting stream resolution
+            if (this.streamUseMaxResolution) {
+                const { width: trackWidth, height: trackHeight } =
+                    sender.track?.getSettings() ?? {};
+
+                // Calculate downscale ratios for width/height
+                let widthRatio = 1;
+                let heightRatio = 1;
+                if (trackWidth && this.streamMaxResolution.width) {
+                    widthRatio = trackWidth / this.streamMaxResolution.width;
+                }
+                if (trackHeight && this.streamMaxResolution.height) {
+                    heightRatio = trackHeight / this.streamMaxResolution.height;
+                }
+
+                // Use the largest ratio to ensure below resolution limit
+                const downscaleRatio = Math.max(1, widthRatio, heightRatio);
+
+                // Multiply existing downscale
+                encoding.scaleResolutionDownBy *= downscaleRatio;
+            }
+
+            await sender.setParameters(params);
+        };
+
+        let stream: MediaStream;
         try {
             // Add screen media stream
-            this.peerConnection.addStream(
+
-                await navigator.mediaDevices.getDisplayMedia({
+            stream = await navigator.mediaDevices.getDisplayMedia({
-                    video: { cursor: "motion" },
+                video: {
-                    audio: false
+                    cursor: "motion",
-                })
+                    frameRate: this.streamMaxFrameRate
-            );
+                },
+                audio: false
+            });
+
+            const [track] = stream.getVideoTracks();
+            pc.addTrack(track, stream);
+            track.addEventListener("ended", () => this.stop());
         } catch (err) {
             logger.error("Failed to add display media stream!", err);
-            this.peerConnection.close();
+            this.stop();
-            this.session?.stop();
+            return;
         }
+
+        /**
+         * Use a video element to get stream resize events and update
+         * scaling parameters.
+         */
+        const video = document.createElement("video");
+        video.srcObject = stream;
+        video.addEventListener("resize", () => applyParameters());
+        video.play();
     }
 }
 
@@ -145,8 +238,12 @@ class MirroringSender {
 if (window.location.protocol !== "moz-extension:") {
     const window_ = window as any;
 
-    new MirroringSender({
+    const sender = new MirroringSender({
         contextTabId: window_.contextTabId,
         receiverDevice: window_.receiverDevice
     });
+
+    window.addEventListener("beforeunload", () => {
+        sender.stop();
+    });
 }

ext/src/defaultOptions.ts

@@ -14,6 +14,11 @@ export interface Options {
     localMediaServerPort: number;
     mirroringEnabled: boolean;
     mirroringAppId: string;
+    mirroringStreamMaxFrameRate: number;
+    mirroringStreamMaxBitRate: number;
+    mirroringStreamDownscaleFactor: number;
+    mirroringStreamMaxResolution: { width?: number; height?: number };
+    mirroringStreamUseMaxResolution: boolean;
     receiverSelectorCloseIfFocusLost: boolean;
     receiverSelectorWaitForConnection: boolean;
     receiverSelectorExpandActive: boolean;
@@ -39,6 +44,11 @@ export default {
     localMediaServerPort: 9555,
     mirroringEnabled: false,
     mirroringAppId: MIRRORING_APP_ID,
+    mirroringStreamMaxFrameRate: 15,
+    mirroringStreamMaxBitRate: 1000000,
+    mirroringStreamDownscaleFactor: 1.0,
+    mirroringStreamMaxResolution: { width: 1920, height: 1080 },
+    mirroringStreamUseMaxResolution: true,
     receiverSelectorCloseIfFocusLost: true,
     receiverSelectorWaitForConnection: true,
     receiverSelectorExpandActive: true,

ext/src/ui/options/Options.svelte

@@ -211,6 +211,121 @@
                         </div>
                     </div>
                 </div>
+
+                <details class="mirroring-stream">
+                    <summary>
+                        {_("optionsMirroringMediaStreamOptions")}
+                    </summary>
+
+                    <div class="mirroring-stream__options category">
+                        <div class="option option--inline scaling-resolution">
+                            <div class="option__control">
+                                <input
+                                    type="checkbox"
+                                    name="scaling"
+                                    id="mirroringStreamUseMaxResolution"
+                                    bind:checked={opts.mirroringStreamUseMaxResolution}
+                                />
+                            </div>
+                            <label
+                                class="option__label"
+                                for="mirroringStreamUseMaxResolution"
+                            >
+                                {_("optionsMirroringStreamMaxResolution")}
+                                <input
+                                    type="number"
+                                    min="1"
+                                    placeholder={_(
+                                        "optionsMirroringStreamMaxResolutionWidthPlaceholder"
+                                    )}
+                                    bind:value={opts
+                                        .mirroringStreamMaxResolution.width}
+                                />
+                                ×
+                                <input
+                                    type="number"
+                                    min="1"
+                                    placeholder={_(
+                                        "optionsMirroringStreamMaxResolutionHeightPlaceholder"
+                                    )}
+                                    bind:value={opts
+                                        .mirroringStreamMaxResolution.height}
+                                />
+                            </label>
+                            <p class="option__description">
+                                {_(
+                                    "optionsMirroringStreamMaxResolutionDescription"
+                                )}
+                            </p>
+                        </div>
+
+                        <div class="option scaling-downscale">
+                            <label
+                                class="option__label"
+                                for="mirroringStreamDownscaleFactor"
+                            >
+                                {_("optionsMirroringStreamDownscaleFactor")}
+                            </label>
+                            <div class="option__control">
+                                <input
+                                    id="mirroringStreamDownscaleFactor"
+                                    type="number"
+                                    required
+                                    min="1"
+                                    step="any"
+                                    bind:value={opts.mirroringStreamDownscaleFactor}
+                                />
+
+                                <p class="option__description">
+                                    {_(
+                                        "optionsMirroringStreamDownscaleFactorDescription"
+                                    )}
+                                </p>
+                            </div>
+                        </div>
+
+                        <div class="option">
+                            <label
+                                class="option__label"
+                                for="mirroringStreamMaxFrameRate"
+                            >
+                                {_("optionsMirroringStreamFrameRate")}
+                            </label>
+                            <div class="option__control">
+                                <input
+                                    id="mirroringStreamMaxFrameRate"
+                                    type="number"
+                                    required
+                                    min="1"
+                                    bind:value={opts.mirroringStreamMaxFrameRate}
+                                />
+                            </div>
+                        </div>
+
+                        <div class="option">
+                            <label
+                                class="option__label"
+                                for="mirroringStreamMaxBitRate"
+                            >
+                                {_("optionsMirroringStreamMaxBitRate")}
+                            </label>
+                            <div class="option__control">
+                                <input
+                                    id="mirroringStreamMaxBitRate"
+                                    type="number"
+                                    required
+                                    min="1"
+                                    bind:value={opts.mirroringStreamMaxBitRate}
+                                />
+                                <p class="option__description">
+                                    {_(
+                                        "optionsMirroringStreamMaxBitRateDescription"
+                                    )}
+                                </p>
+                            </div>
+                        </div>
+                    </div>
+                </details>
             </fieldset>
         {/if}
 

ext/src/ui/options/styles/index.css

@@ -1,5 +1,10 @@
 #root {
-    padding: 20px 10px;
+    --overlay-color: rgba(255, 255, 255, 0.05);
+}
+@media (prefers-color-scheme: dark) {
+    :root {
+        --overlay-color: rgba(0, 0, 0, 0.1);
+    }
 }
 
 body {
@@ -7,6 +12,7 @@ body {
     color: var(--box-color) !important;
     font-size: 13px;
     overflow-y: hidden;
+    padding: 20px 10px;
 }
 
 a {
@@ -230,6 +236,16 @@ input:placeholder-shown {
     grid-column: span 2;
     width: 100%;
 }
+.category > details {
+    display: contents;
+}
+.category > details > summary {
+    grid-column: 2;
+    margin-top: 5px;
+}
+.category > details:not([open]) > summary ~ * {
+    display: none;
+}
 
 .category__name {
     float: left;
@@ -315,6 +331,17 @@ fieldset:disabled .option__description {
     margin-inline-start: initial;
 }
 
+.mirroring-stream > summary {
+    color: var(--secondary-color);
+}
+.category.mirroring-stream__options {
+    background-color: var(--overlay-color);
+    border-radius: 4px;
+    grid-column: 2;
+    grid-template-columns: max-content 1fr;
+    padding: 10px;
+}
+
 .whitelist {
     background-color: var(--box-background);
     border: 1px solid var(--border-color);
@@ -354,7 +381,7 @@ fieldset:disabled .option__description {
 }
 
 .whitelist__item:nth-child(even) {
-    background-color: rgba(0, 0, 0, 0.1);
+    background-color: var(--overlay-color);
 }
 
 .whitelist__item--selected {
@@ -435,8 +462,7 @@ input[id^="customUserAgentString-"] {
     width: -moz-available;
 }
 
-@media (prefers-color-scheme: dark) {
+.scaling-resolution input[type="number"],
-    .whitelist__item:nth-child(even) {
+.scaling-downscale input[type="number"] {
-        background-color: rgba(255, 255, 255, 0.05);
+    width: 75px;
-    }
 }