From 314a1d2031ac517c90aabcff9cb02f30bb51e314 Mon Sep 17 00:00:00 2001 From: hensm Date: Thu, 1 Sep 2022 12:23:03 +0100 Subject: [PATCH] Restore mirroring sender functionality to level before SDK refactoring --- IMPLEMENTATION.md | 2 + ext/src/background/castManager.ts | 53 ++++- ext/src/cast/contentBridge.ts | 4 +- ext/src/cast/export.ts | 86 ++++++-- ext/src/cast/senders/media.ts | 10 +- ext/src/cast/senders/mirroring.ts | 343 ++++++++++++++++-------------- ext/src/messaging.ts | 2 + 7 files changed, 303 insertions(+), 197 deletions(-) diff --git a/IMPLEMENTATION.md b/IMPLEMENTATION.md index 2c8cf2a..02db2c6 100644 --- a/IMPLEMENTATION.md +++ b/IMPLEMENTATION.md @@ -57,6 +57,8 @@ For an instance created for an extension script: - If **content/extension page**: The `contentBridge.ts` script is imported as a module, with the usual side-effects of creating a messaging connection to the Cast Manager and hooking up page messaging (as described for page script instances). 3. Listeners are added for the `cast:instanceCreated` message, so that the `ensureInit` function can resolve its promise and provide a Cast Manager port after initialization. +Extension sender apps are considered to be trusted by the cast manager and are granted additional privileges. They can bypass the receiver selection step when requesting a session by providing a receiver device when initialising the SDK via `ensureInit`. + #### All contexts The process now continues identically for all contexts: diff --git a/ext/src/background/castManager.ts b/ext/src/background/castManager.ts index 95db785..14599a6 100644 --- a/ext/src/background/castManager.ts +++ b/ext/src/background/castManager.ts @@ -47,6 +47,9 @@ export interface CastInstance { contentPort: AnyPort; contentContext?: ContentContext; + /** From an extension-source, grants additional permissions. */ + isTrusted: boolean; + /** ApiConfig provided on initialization. */ apiConfig?: ApiConfig; /** Established session details. */ @@ -58,10 +61,12 @@ async function createCastInstance(opts: { bridgePort?: Port; contentPort: AnyPort; contentContext?: { tabId: number; frameId?: number }; + isTrusted?: boolean; }) { const instance: CastInstance = { bridgePort: opts.bridgePort ?? (await bridge.connect()), - contentPort: opts.contentPort + contentPort: opts.contentPort, + isTrusted: opts.isTrusted ?? false }; /** @@ -113,6 +118,9 @@ const castManager = new (class { messaging.onConnect.addListener(async port => { if (port.name === "cast") { this.createInstance(port); + } else if (port.name === "trusted-cast") { + // Create trusted instance + this.createInstance(port, undefined, true); } }); @@ -161,10 +169,14 @@ const castManager = new (class { * Creates a cast instance with a given port and connects messaging * correctly depending on the type of port. */ - async createInstance(port: AnyPort, contentContext?: ContentContext) { + async createInstance( + port: AnyPort, + contentContext?: ContentContext, + isTrusted?: boolean + ) { const instance = await (port instanceof MessagePort ? this.createInstanceFromBackground(port, contentContext) - : this.createInstanceFromContent(port)); + : this.createInstanceFromContent(port, isTrusted)); this.activeInstances.add(instance); @@ -184,7 +196,8 @@ const castManager = new (class { const instance = await createCastInstance({ bridgePort: await bridge.connect(), contentPort, - contentContext + contentContext, + isTrusted: true }); // Ensure only one instance per context @@ -221,7 +234,8 @@ const castManager = new (class { * Creates a cast instance with a WebExtension `Port` content port. */ private async createInstanceFromContent( - contentPort: Port + contentPort: Port, + isTrusted?: boolean ): Promise { if ( contentPort.sender?.tab?.id === undefined || @@ -246,7 +260,7 @@ const castManager = new (class { } } - const instance = await createCastInstance({ contentPort }); + const instance = await createCastInstance({ contentPort, isTrusted }); // cast instance -> (any) const onContentPortMessage = (message: Message) => { @@ -362,7 +376,27 @@ const castManager = new (class { // User has triggered receiver selection via the cast API case "main:requestSession": { - const { sessionRequest } = message.data; + const { sessionRequest, receiverDevice } = message.data; + + // Handle trusted instance receiver selection bypass + if (receiverDevice) { + if (!instance.isTrusted) { + logger.error( + "Cast instance not trusted to bypass receiver selection!" + ); + break; + } + + instance.bridgePort.postMessage({ + subject: "bridge:createCastSession", + data: { + appId: sessionRequest.appId, + receiverDevice + } + }); + + break; + } try { const selection = await getReceiverSelection({ @@ -485,8 +519,9 @@ const castManager = new (class { case ReceiverSelectorMediaType.Screen: await browser.tabs.executeScript(contentContext.tabId, { code: stringify` - window.selectedMedia = ${selection.mediaType}; - window.selectedReceiver = ${selection.receiverDevice}; + window.mirroringMediaType = ${selection.mediaType}; + window.receiverDevice = ${selection.receiverDevice}; + window.contextTabId = ${contentContext.tabId}; `, frameId: contentContext.frameId }); diff --git a/ext/src/cast/contentBridge.ts b/ext/src/cast/contentBridge.ts index 4275b18..dde4481 100644 --- a/ext/src/cast/contentBridge.ts +++ b/ext/src/cast/contentBridge.ts @@ -1,8 +1,8 @@ import messaging, { Message } from "../messaging"; import pageMessenging from "./pageMessenging"; -// Message port to background script -export const managerPort = messaging.connect({ name: "cast" }); +// Message port to cast manager in background script +const managerPort = messaging.connect({ name: "cast" }); const forwardToPage = (message: Message) => { pageMessenging.extension.sendMessage(message); diff --git a/ext/src/cast/export.ts b/ext/src/cast/export.ts index f4becfe..56f669d 100644 --- a/ext/src/cast/export.ts +++ b/ext/src/cast/export.ts @@ -2,7 +2,8 @@ "use strict"; import type { TypedMessagePort } from "../lib/TypedMessagePort"; -import type { Message } from "../messaging"; +import messaging, { Message } from "../messaging"; +import type { ReceiverDevice } from "../types"; import pageMessenging from "./pageMessenging"; import CastSDK from "./sdk"; @@ -14,17 +15,23 @@ let existingInstance = new CastSDK(); export default existingInstance; +interface EnsureInitOpts { + contextTabId?: number; + /** Skip receiver selection. */ + receiverDevice?: ReceiverDevice; +} + /** * To support exporting the API from a module, we need to retain the * MessageChannel-based pageMessaging layer despite not crossing any * context boundaries. * * The ensureInit function creates a messaging connection to the - * castManager, hooks it up to the pageMessaging layer and also provides - * a messaging port so consumers of this module can communicate with the - * castManager. + * cast manager, hooks it up to the pageMessaging layer and also + * provides a messaging port so consumers of this module can communicate + * with the cast manager. */ -export function ensureInit(contextTabId?: number): Promise { +export function ensureInit(opts: EnsureInitOpts): Promise { return new Promise(async (resolve, reject) => { // If already initialized if (existingPort) { @@ -43,26 +50,26 @@ export function ensureInit(contextTabId?: number): Promise { ); /** - * port1 will handle castManager messages. + * port1 will handle cast manager messages. * port2 will handle cast instance messages. */ const { port1: managerPort, port2: instancePort } = new MessageChannel(); /** - * Provide castManager with a port to send messages to + * Provide cast manager with a port to send messages to * cast instance. */ - if (contextTabId) { + if (opts.contextTabId) { await castManager.createInstance(instancePort, { - tabId: contextTabId, + tabId: opts.contextTabId, frameId: 0 }); } else { await castManager.createInstance(instancePort); } - // castManager -> cast instance + // cast manager -> cast instance managerPort.addEventListener("message", ev => { const message = ev.data as Message; if (message.subject === "cast:instanceCreated") { @@ -77,30 +84,67 @@ export function ensureInit(contextTabId?: number): Promise { }); managerPort.start(); - // Cast instance -> castManager + // Cast instance -> cast manager pageMessenging.extension.addListener(message => { + // Skip receiver selection + if (opts.receiverDevice) { + message = rewriteTrustedRequestSession( + message, + opts.receiverDevice + ); + } + managerPort.postMessage(message); }); } else { - // Let contentBridge hook up pageMessaging - const { managerPort: backgroundPort } = await import( - "./contentBridge" - ); - existingPort = pageMessenging.page.messagePort; + const managerPort = messaging.connect({ name: "trusted-cast" }); - backgroundPort.onMessage.addListener(function onManagerMessage( - message: Message - ) { + // Cast manager -> cast instance + managerPort.onMessage.addListener(message => { if (message.subject === "cast:instanceCreated") { if (message.data.isAvailable) { resolve(pageMessenging.page.messagePort); } else { reject(); } - - backgroundPort.onMessage.removeListener(onManagerMessage); } + + pageMessenging.extension.sendMessage(message); }); + + // Cast instance -> cast manager + pageMessenging.extension.addListener(message => { + // Skip receiver selection + if (opts.receiverDevice) { + message = rewriteTrustedRequestSession( + message, + opts.receiverDevice + ); + } + + managerPort.postMessage(message); + }); + + managerPort.onDisconnect.addListener(() => { + pageMessenging.extension.close(); + }); + + existingPort = pageMessenging.page.messagePort; } }); } + +/** + * If a receiver device was passed to `ensureInit`, messages to the cast + * manager will be passed through this function and the receiver device + * will be added to the message payload. This tells the cast manager to + * skip receiver selection when requesting a session. + */ +function rewriteTrustedRequestSession( + message: Message, + receiverDevice: ReceiverDevice +) { + if (message.subject !== "main:requestSession") return message; + message.data.receiverDevice = receiverDevice; + return message; +} diff --git a/ext/src/cast/senders/media.ts b/ext/src/cast/senders/media.ts index dd8039c..12265ad 100644 --- a/ext/src/cast/senders/media.ts +++ b/ext/src/cast/senders/media.ts @@ -30,6 +30,8 @@ export default class MediaSender { private isLocalMedia = false; private isLocalMediaEnabled = false; + private wasSessionRequested = false; + // Cast API objects private session?: Session; private media?: Media; @@ -49,7 +51,7 @@ export default class MediaSender { private async init() { try { - this.port = await ensureInit(this.contextTabId); + this.port = await ensureInit({ contextTabId: this.contextTabId }); } catch (err) { logger.error("Failed to initialize cast API", err); } @@ -96,12 +98,12 @@ export default class MediaSender { // Unused } private receiverListener(availability: ReceiverAvailability) { - // Already have session - if (this.session) return; + if (this.wasSessionRequested) return; + this.wasSessionRequested = false; if (availability === cast.ReceiverAvailability.AVAILABLE) { cast.requestSession( - (session: Session) => { + session => { this.session = session; this.loadMedia(); }, diff --git a/ext/src/cast/senders/mirroring.ts b/ext/src/cast/senders/mirroring.ts index a9f2e23..b2237b8 100644 --- a/ext/src/cast/senders/mirroring.ts +++ b/ext/src/cast/senders/mirroring.ts @@ -1,199 +1,220 @@ "use strict"; import options from "../../lib/options"; -import cast, { ensureInit } from "../export"; +import { Logger } from "../../lib/logger"; import { ReceiverDevice, ReceiverSelectorMediaType } from "../../types"; import type Session from "../sdk/Session"; +import cast, { ensureInit } from "../export"; +import type { ReceiverAvailability } from "../sdk/enums"; -// Variables passed from background -const { - selectedMedia, - selectedReceiver -}: { - selectedMedia: ReceiverSelectorMediaType; - selectedReceiver: ReceiverDevice; -} = window as any; +const logger = new Logger("fx_cast [mirroring sender]"); -const FX_CAST_RECEIVER_APP_NAMESPACE = "urn:x-cast:fx_cast"; +const NS_FX_CAST = "urn:x-cast:fx_cast"; -let session: Session; -let wasSessionRequested = false; +type MirroringAppMessage = + | { subject: "peerConnectionOffer"; data: RTCSessionDescriptionInit } + | { subject: "peerConnectionAnswer"; data: RTCSessionDescriptionInit } + | { subject: "iceCandidate"; data: RTCIceCandidateInit } + | { subject: "close" }; -let peerConnection: RTCPeerConnection; +type MirroringMediaType = + | ReceiverSelectorMediaType.Tab + | ReceiverSelectorMediaType.Screen; -/** - * Sends a message to the fx_cast app running on the - * receiver device. - */ -function sendAppMessage(subject: string, data: unknown) { - if (!session) { - return; - } - - session.sendMessage(FX_CAST_RECEIVER_APP_NAMESPACE, { - subject, - data - }); +interface MirroringSenderOpts { + mirroringMediaType: MirroringMediaType; + contextTabId?: number; + receiverDevice?: ReceiverDevice; } -window.addEventListener("beforeunload", () => { - sendAppMessage("close", null); -}); +class MirroringSender { + private mirroringMediaType: MirroringMediaType; + private contextTabId?: number; + private receiverDevice?: ReceiverDevice; -async function onRequestSessionSuccess(newSession: Session) { - cast.logMessage("onRequestSessionSuccess"); + private session?: Session; + private wasSessionRequested = false; - session = newSession; - session.addMessageListener( - FX_CAST_RECEIVER_APP_NAMESPACE, - async (_namespace, message) => { - const { subject, data } = JSON.parse(message); + private peerConnection?: RTCPeerConnection; - switch (subject) { - case "peerConnectionAnswer": { - peerConnection.setRemoteDescription(data); - break; - } - case "iceCandidate": { - peerConnection.addIceCandidate(data); - break; - } - } + constructor(opts: MirroringSenderOpts) { + this.mirroringMediaType = opts.mirroringMediaType; + this.contextTabId = opts.contextTabId; + this.receiverDevice = opts.receiverDevice; + + this.init(); + } + + private async init() { + try { + ensureInit({ + contextTabId: this.contextTabId, + receiverDevice: this.receiverDevice + }); + } catch (err) { + logger.error("Failed to initialize cast API", err); } - ); - peerConnection = new RTCPeerConnection(); - peerConnection.addEventListener("icecandidate", ev => { - sendAppMessage("iceCandidate", ev.candidate); - }); + const mirroringAppId = await options.get("mirroringAppId"); + const sessionRequest = new cast.SessionRequest(mirroringAppId); - switch (selectedMedia) { - case ReceiverSelectorMediaType.Tab: { - const canvas = document.createElement("canvas"); - const ctx = canvas.getContext("2d"); + const apiConfig = new cast.ApiConfig( + sessionRequest, + this.sessionListener, + this.receiverListener + ); - // Shouldn't be possible - if (!ctx) { - break; + cast.initialize(apiConfig); + } + + private sessionListener() { + // Unused + } + private receiverListener = (availability: ReceiverAvailability) => { + if (this.wasSessionRequested) return; + this.wasSessionRequested = true; + + if (availability === cast.ReceiverAvailability.AVAILABLE) { + cast.requestSession( + session => { + this.session = session; + this.createMirroringConnection(); + }, + err => { + logger.error("Session request failed", err); + } + ); + } + }; + + private sendMirroringAppMessage(message: MirroringAppMessage) { + if (!this.session) return; + this.session.sendMessage(NS_FX_CAST, message); + } + + private async createMirroringConnection() { + this.session?.addMessageListener(NS_FX_CAST, async (_ns, message) => { + const parsedMessage = JSON.parse(message) as MirroringAppMessage; + switch (parsedMessage.subject) { + case "peerConnectionAnswer": + this.peerConnection?.setRemoteDescription( + parsedMessage.data + ); + break; + case "iceCandidate": + this.peerConnection?.addIceCandidate(parsedMessage.data); + break; } + }); - // Set initial size + this.peerConnection = new RTCPeerConnection(); + this.peerConnection.addEventListener("icecandidate", ev => { + if (!ev.candidate) return; + this.sendMirroringAppMessage({ + subject: "iceCandidate", + data: ev.candidate + }); + }); + + switch (this.mirroringMediaType) { + case ReceiverSelectorMediaType.Tab: + this.peerConnection.addStream(this.getTabStream()); + break; + case ReceiverSelectorMediaType.Screen: + this.peerConnection.addStream(await this.getScreenStream()); + break; + } + + const offer = await this.peerConnection.createOffer(); + await this.peerConnection.setLocalDescription(offer); + + this.sendMirroringAppMessage({ + subject: "peerConnectionOffer", + data: offer + }); + } + + private getTabStream() { + const canvas = document.createElement("canvas"); + const ctx = canvas.getContext("2d"); + if (!ctx) { + throw logger.error("Failed to get tab canvas context!"); + } + + // Set initial size + canvas.width = window.innerWidth * window.devicePixelRatio; + canvas.height = window.innerHeight * window.devicePixelRatio; + ctx.scale(window.devicePixelRatio, window.devicePixelRatio); + + // Resize canvas whenever the window resizes + window.addEventListener("resize", () => { canvas.width = window.innerWidth * window.devicePixelRatio; canvas.height = window.innerHeight * window.devicePixelRatio; + ctx.setTransform(1, 0, 0, 1, 0, 0); ctx.scale(window.devicePixelRatio, window.devicePixelRatio); + }); - // Resize canvas whenever the window resizes - window.addEventListener("resize", () => { - canvas.width = window.innerWidth * window.devicePixelRatio; - canvas.height = window.innerHeight * window.devicePixelRatio; - ctx.setTransform(1, 0, 0, 1, 0, 0); - ctx.scale(window.devicePixelRatio, window.devicePixelRatio); - }); + const drawFlags = + ctx.DRAWWINDOW_DRAW_CARET | + ctx.DRAWWINDOW_DRAW_VIEW | + ctx.DRAWWINDOW_ASYNC_DECODE_IMAGES | + ctx.DRAWWINDOW_USE_WIDGET_LAYERS; - const drawFlags = - ctx.DRAWWINDOW_DRAW_CARET | - ctx.DRAWWINDOW_DRAW_VIEW | - ctx.DRAWWINDOW_ASYNC_DECODE_IMAGES | - ctx.DRAWWINDOW_USE_WIDGET_LAYERS; + let lastFrame: DOMHighResTimeStamp; + window.requestAnimationFrame(function draw(now: DOMHighResTimeStamp) { + if (!lastFrame) { + lastFrame = now; + } - let lastFrame: DOMHighResTimeStamp; - window.requestAnimationFrame(function draw( - now: DOMHighResTimeStamp - ) { - if (!lastFrame) { - lastFrame = now; - } + if (now - lastFrame > 1000 / 30) { + ctx.drawWindow( + window, // window + 0, + 0, // x, y + canvas.width, // w + canvas.height, // h + "white", // bgColor + drawFlags + ); // flags - if (now - lastFrame > 1000 / 30) { - ctx.drawWindow( - window, // window - 0, - 0, // x, y - canvas.width, // w - canvas.height, // h - "white", // bgColor - drawFlags - ); // flags + lastFrame = now; + } - lastFrame = now; - } + window.requestAnimationFrame(draw); + }); - window.requestAnimationFrame(draw); - }); - - /** - * Capture video stream from canvas and feed into the RTC - * connection. - */ - peerConnection.addStream(canvas.captureStream()); - - break; - } - - case ReceiverSelectorMediaType.Screen: { - const stream = await navigator.mediaDevices.getDisplayMedia({ - video: { cursor: "motion" }, - audio: false - }); - - peerConnection.addStream(stream); - - break; - } + return canvas.captureStream(); } - // Create SDP offer and set locally - const offer = await peerConnection.createOffer(); - await peerConnection.setLocalDescription(offer); - - // Send local offer to receiver app - sendAppMessage("peerConnectionOffer", offer); -} - -function receiverListener(availability: string) { - cast.logMessage("receiverListener"); - - if (wasSessionRequested) { - return; - } - - if (availability === cast.ReceiverAvailability.AVAILABLE) { - wasSessionRequested = true; - cast.requestSession( - onRequestSessionSuccess, - onRequestSessionError, - undefined, - selectedReceiver - ); + private getScreenStream() { + return new Promise(resolve => { + window.addEventListener( + "click", + () => { + resolve( + navigator.mediaDevices.getDisplayMedia({ + video: { cursor: "motion" }, + audio: false + }) + ); + }, + { once: true } + ); + }); } } -function onRequestSessionError() { - cast.logMessage("onRequestSessionError"); -} -function sessionListener() { - cast.logMessage("sessionListener"); -} -function onInitializeSuccess() { - cast.logMessage("onInitializeSuccess"); -} -function onInitializeError() { - cast.logMessage("onInitializeError"); -} +/** + * If loaded as a content script, opts are stored on the window object. + */ +if (window.location.protocol !== "moz-extension:") { + const window_ = window as any; -ensureInit().then(async () => { - const mirroringAppId = await options.get("mirroringAppId"); - const sessionRequest = new cast.SessionRequest(mirroringAppId); - - const apiConfig = new cast.ApiConfig( - sessionRequest, - sessionListener, - receiverListener, - undefined, - undefined - ); - - cast.initialize(apiConfig, onInitializeSuccess, onInitializeError); -}); + new MirroringSender({ + mirroringMediaType: window_.mirroringMediaType, + contextTabId: window_.contextTabId, + receiverDevice: window_.receiverDevice + }); +} diff --git a/ext/src/messaging.ts b/ext/src/messaging.ts index da7cd9b..65a1132 100644 --- a/ext/src/messaging.ts +++ b/ext/src/messaging.ts @@ -86,6 +86,8 @@ type ExtMessageDefinitions = { */ "main:requestSession": { sessionRequest: SessionRequest; + /** Skip receiver selection (allowed for trusted instances only). */ + receiverDevice?: ReceiverDevice; }; /** Return message to the cast API when a selection is cancelled. */ "cast:sessionRequestCancelled": undefined;