From eac20dfe4a9633004ffd0571f1e2089929b9615e Mon Sep 17 00:00:00 2001 From: hensm Date: Wed, 7 Sep 2022 15:47:40 +0100 Subject: [PATCH] Improve screen mirroring performance and add stream encoding options --- ext/src/_locales/en/messages.json | 40 ++++++++ ext/src/cast/senders/mirroring.ts | 143 +++++++++++++++++++++++----- ext/src/defaultOptions.ts | 10 ++ ext/src/ui/options/Options.svelte | 115 ++++++++++++++++++++++ ext/src/ui/options/styles/index.css | 38 ++++++-- 5 files changed, 317 insertions(+), 29 deletions(-) diff --git a/ext/src/_locales/en/messages.json b/ext/src/_locales/en/messages.json index 8dc5a81..bbe1347 100755 --- a/ext/src/_locales/en/messages.json +++ b/ext/src/_locales/en/messages.json @@ -498,6 +498,46 @@ "message": "App ID for a registered Chromecast receiver application. Advanced use only. Must be compatible with the default app (see GitHub repo).", "description": "Mirroring app ID option description." }, + "optionsMirroringMediaStreamOptions": { + "message": "Media stream encoding options", + "description": "Options page mirroring category description." + }, + "optionsMirroringStreamFrameRate": { + "message": "Max frame rate:", + "description": "Mirroring stream max frame rate option label." + }, + "optionsMirroringStreamMaxBitRate": { + "message": "Max bitrate:", + "description": "Mirroring stream max bit rate option label." + }, + "optionsMirroringStreamMaxBitRateDescription": { + "message": "Maximum bitrate in bits per second.", + "description": "Mirroring stream max bit rate option description." + }, + "optionsMirroringStreamDownscaleFactor": { + "message": "Downscale factor:", + "description": "Mirroring stream downscale factor option label." + }, + "optionsMirroringStreamDownscaleFactorDescription": { + "message": "Factor by which to scale down the video stream e.g. a factor of 2.0 would result in a video 1/4 the size. ", + "description": "Mirroring stream downscale factor option description." + }, + "optionsMirroringStreamMaxResolution": { + "message": "Limit resolution to", + "description": "Mirroring stream resolution option label." + }, + "optionsMirroringStreamMaxResolutionDescription": { + "message": "Limits the maximum video stream resolution whilst maintaining the source aspect ratio.", + "description": "Mirroring stream resolution option description." + }, + "optionsMirroringStreamMaxResolutionWidthPlaceholder": { + "message": "Width", + "description": "Max resolution width input placeholder." + }, + "optionsMirroringStreamMaxResolutionHeightPlaceholder": { + "message": "Height", + "description": "Max resolution height input placeholder." + }, "optionsOptionRecommended": { "message": "recommended", diff --git a/ext/src/cast/senders/mirroring.ts b/ext/src/cast/senders/mirroring.ts index 2c7bc0d..ba6f692 100644 --- a/ext/src/cast/senders/mirroring.ts +++ b/ext/src/cast/senders/mirroring.ts @@ -30,7 +30,14 @@ class MirroringSender { private session?: Session; private wasSessionRequested = false; - private peerConnection?: RTCPeerConnection; + private peerConnection: Optional; + + // Stream opts + private streamMaxFrameRate = 1; + private streamMaxBitRate = 1; + private streamDownscaleFactor = 1; + private streamUseMaxResolution = false; + private streamMaxResolution: { width?: number; height?: number } = {}; constructor(opts: MirroringSenderOpts) { this.contextTabId = opts.contextTabId; @@ -49,7 +56,21 @@ class MirroringSender { logger.error("Failed to initialize cast API", err); } - const mirroringAppId = await options.get("mirroringAppId"); + const { + mirroringAppId, + mirroringStreamMaxFrameRate, + mirroringStreamMaxBitRate, + mirroringStreamDownscaleFactor, + mirroringStreamUseMaxResolution, + mirroringStreamMaxResolution + } = await options.getAll(); + + this.streamMaxFrameRate = mirroringStreamMaxFrameRate; + this.streamMaxBitRate = mirroringStreamMaxBitRate; + this.streamDownscaleFactor = mirroringStreamDownscaleFactor; + this.streamUseMaxResolution = mirroringStreamUseMaxResolution; + this.streamMaxResolution = mirroringStreamMaxResolution; + const sessionRequest = new cast.SessionRequest(mirroringAppId); const apiConfig = new cast.ApiConfig( @@ -61,6 +82,11 @@ class MirroringSender { cast.initialize(apiConfig); } + stop() { + this.peerConnection?.close(); + this.session?.stop(); + } + private sessionListener() { // Unused } @@ -87,27 +113,24 @@ class MirroringSender { } private async createMirroringConnection() { + const pc = new RTCPeerConnection(); + this.peerConnection = pc; + this.session?.addMessageListener(NS_FX_CAST, async (_ns, message) => { const parsedMessage = JSON.parse(message) as MirroringAppMessage; switch (parsedMessage.subject) { case "peerConnectionAnswer": - this.peerConnection?.setRemoteDescription( - parsedMessage.data - ); + pc.setRemoteDescription(parsedMessage.data); break; case "iceCandidate": - this.peerConnection?.addIceCandidate(parsedMessage.data); + pc.addIceCandidate(parsedMessage.data); break; } }); - this.peerConnection = new RTCPeerConnection(); - - this.peerConnection.addEventListener("negotiationneeded", async () => { - if (!this.peerConnection) return; - - const offer = await this.peerConnection.createOffer(); - await this.peerConnection.setLocalDescription(offer); + pc.addEventListener("negotiationneeded", async () => { + const offer = await pc.createOffer(); + await pc.setLocalDescription(offer); this.sendMirroringAppMessage({ subject: "peerConnectionOffer", @@ -115,7 +138,7 @@ class MirroringSender { }); }); - this.peerConnection.addEventListener("icecandidate", ev => { + pc.addEventListener("icecandidate", ev => { if (!ev.candidate) return; this.sendMirroringAppMessage({ subject: "iceCandidate", @@ -123,19 +146,89 @@ class MirroringSender { }); }); + // Connection listener + pc.addEventListener("iceconnectionstatechange", async () => { + if (pc.iceConnectionState !== "connected") { + return; + } + + applyParameters(); + }); + + /** Applies stream encoding parameters. */ + const applyParameters = async () => { + // Set stream encoding parameters + const [sender] = pc.getSenders(); + const params = sender.getParameters(); + if (!params.encodings) { + params.encodings = [{}]; + } + + const [encoding] = params.encodings; + + if (!(encoding as any).maxFramerate) { + (encoding as any).maxFramerate = this.streamMaxFrameRate; + } + if (!encoding.maxBitrate) { + encoding.maxBitrate = this.streamMaxBitRate; + } + + encoding.scaleResolutionDownBy = this.streamDownscaleFactor; + + // Handle limiting stream resolution + if (this.streamUseMaxResolution) { + const { width: trackWidth, height: trackHeight } = + sender.track?.getSettings() ?? {}; + + // Calculate downscale ratios for width/height + let widthRatio = 1; + let heightRatio = 1; + if (trackWidth && this.streamMaxResolution.width) { + widthRatio = trackWidth / this.streamMaxResolution.width; + } + if (trackHeight && this.streamMaxResolution.height) { + heightRatio = trackHeight / this.streamMaxResolution.height; + } + + // Use the largest ratio to ensure below resolution limit + const downscaleRatio = Math.max(1, widthRatio, heightRatio); + + // Multiply existing downscale + encoding.scaleResolutionDownBy *= downscaleRatio; + } + + await sender.setParameters(params); + }; + + let stream: MediaStream; try { // Add screen media stream - this.peerConnection.addStream( - await navigator.mediaDevices.getDisplayMedia({ - video: { cursor: "motion" }, - audio: false - }) - ); + + stream = await navigator.mediaDevices.getDisplayMedia({ + video: { + cursor: "motion", + frameRate: this.streamMaxFrameRate + }, + audio: false + }); + + const [track] = stream.getVideoTracks(); + pc.addTrack(track, stream); + track.addEventListener("ended", () => this.stop()); } catch (err) { logger.error("Failed to add display media stream!", err); - this.peerConnection.close(); - this.session?.stop(); + this.stop(); + return; } + + /** + * Use a video element to get stream resize events and update + * scaling parameters. + */ + const video = document.createElement("video"); + video.srcObject = stream; + video.addEventListener("resize", () => applyParameters()); + video.play(); } } @@ -145,8 +238,12 @@ class MirroringSender { if (window.location.protocol !== "moz-extension:") { const window_ = window as any; - new MirroringSender({ + const sender = new MirroringSender({ contextTabId: window_.contextTabId, receiverDevice: window_.receiverDevice }); + + window.addEventListener("beforeunload", () => { + sender.stop(); + }); } diff --git a/ext/src/defaultOptions.ts b/ext/src/defaultOptions.ts index 416bd5b..2c6938e 100644 --- a/ext/src/defaultOptions.ts +++ b/ext/src/defaultOptions.ts @@ -14,6 +14,11 @@ export interface Options { localMediaServerPort: number; mirroringEnabled: boolean; mirroringAppId: string; + mirroringStreamMaxFrameRate: number; + mirroringStreamMaxBitRate: number; + mirroringStreamDownscaleFactor: number; + mirroringStreamMaxResolution: { width?: number; height?: number }; + mirroringStreamUseMaxResolution: boolean; receiverSelectorCloseIfFocusLost: boolean; receiverSelectorWaitForConnection: boolean; receiverSelectorExpandActive: boolean; @@ -39,6 +44,11 @@ export default { localMediaServerPort: 9555, mirroringEnabled: false, mirroringAppId: MIRRORING_APP_ID, + mirroringStreamMaxFrameRate: 15, + mirroringStreamMaxBitRate: 1000000, + mirroringStreamDownscaleFactor: 1.0, + mirroringStreamMaxResolution: { width: 1920, height: 1080 }, + mirroringStreamUseMaxResolution: true, receiverSelectorCloseIfFocusLost: true, receiverSelectorWaitForConnection: true, receiverSelectorExpandActive: true, diff --git a/ext/src/ui/options/Options.svelte b/ext/src/ui/options/Options.svelte index 53e4ef8..259a42d 100644 --- a/ext/src/ui/options/Options.svelte +++ b/ext/src/ui/options/Options.svelte @@ -211,6 +211,121 @@ + +
+ + {_("optionsMirroringMediaStreamOptions")} + + +
+
+
+ +
+ +

+ {_( + "optionsMirroringStreamMaxResolutionDescription" + )} +

+
+ +
+ +
+ + +

+ {_( + "optionsMirroringStreamDownscaleFactorDescription" + )} +

+
+
+ +
+ +
+ +
+
+ +
+ +
+ +

+ {_( + "optionsMirroringStreamMaxBitRateDescription" + )} +

+
+
+
+
{/if} diff --git a/ext/src/ui/options/styles/index.css b/ext/src/ui/options/styles/index.css index 6558b9b..cae67db 100644 --- a/ext/src/ui/options/styles/index.css +++ b/ext/src/ui/options/styles/index.css @@ -1,5 +1,10 @@ #root { - padding: 20px 10px; + --overlay-color: rgba(255, 255, 255, 0.05); +} +@media (prefers-color-scheme: dark) { + :root { + --overlay-color: rgba(0, 0, 0, 0.1); + } } body { @@ -7,6 +12,7 @@ body { color: var(--box-color) !important; font-size: 13px; overflow-y: hidden; + padding: 20px 10px; } a { @@ -230,6 +236,16 @@ input:placeholder-shown { grid-column: span 2; width: 100%; } +.category > details { + display: contents; +} +.category > details > summary { + grid-column: 2; + margin-top: 5px; +} +.category > details:not([open]) > summary ~ * { + display: none; +} .category__name { float: left; @@ -315,6 +331,17 @@ fieldset:disabled .option__description { margin-inline-start: initial; } +.mirroring-stream > summary { + color: var(--secondary-color); +} +.category.mirroring-stream__options { + background-color: var(--overlay-color); + border-radius: 4px; + grid-column: 2; + grid-template-columns: max-content 1fr; + padding: 10px; +} + .whitelist { background-color: var(--box-background); border: 1px solid var(--border-color); @@ -354,7 +381,7 @@ fieldset:disabled .option__description { } .whitelist__item:nth-child(even) { - background-color: rgba(0, 0, 0, 0.1); + background-color: var(--overlay-color); } .whitelist__item--selected { @@ -435,8 +462,7 @@ input[id^="customUserAgentString-"] { width: -moz-available; } -@media (prefers-color-scheme: dark) { - .whitelist__item:nth-child(even) { - background-color: rgba(255, 255, 255, 0.05); - } +.scaling-resolution input[type="number"], +.scaling-downscale input[type="number"] { + width: 75px; }