diff --git a/CHANGELOG.md b/CHANGELOG.md index e47a117..ab28480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,22 @@ Release notes follow the [keep a changelog](https://keepachangelog.com/en/1.0.0/ ### Added - Added `AirConsole.requestPhotonEngineAuth()` and `onPhotonEngineAuth(ticket)` so screens can authenticate with Photon Custom Authentication. +- `AirConsole.getUserMedia` asks the platform which audio input a stream should use before handing it to the game + (`requestPreferredAudioInputDevice` / `preferredAudioInputDevice`), so car head units can keep games on the phone's + built-in microphone. The first request opens a stream, then keeps it, opens a fresh one if it ended, or switches to + the preferred device; later requests ask first, then open once with the device merged in as a non-exact `deviceId`. +- The API reports the audio inputs and the active device each time `getUserMedia` resolves + (`audioInputDevicesReported`), and failures the platform would not otherwise learn of, a timeout or an open error + (`userMediaRequestFailed`). +- The preferred audio input device exchange runs only when the platform announces it in `ready` + (`audioInputExchangeEnabled: true`); otherwise `getUserMedia` behaves as before. + +### Changed + +- With the exchange, the `getUserMedia` timeout is 60 s of unpaused time: it stops while the screen is paused. + Without it, the timeout stays 45 s. +- With the exchange, `destroy()` ends a pending `getUserMedia` request like a timeout: it stops any stream the request + holds and reports `userMediaRequestFailed` before rejecting. ## [1.11.0] - 2026-07-07 diff --git a/airconsole-1.11.0.js b/airconsole-1.11.0.js index 38ae1ae..fcd829e 100644 --- a/airconsole-1.11.0.js +++ b/airconsole-1.11.0.js @@ -806,16 +806,33 @@ AirConsole.prototype.onUserMediaAccessDenied = function (device_id) {}; */ const mediaPermissionCallbacks_ = new WeakMap(); +/** + * How long a getUserMedia request may stay unsettled, counted in unpaused time: the clock stops while the screen is + * paused. It covers everything a request waits on: the car-connection notice, the permission flow, the preferred + * audio input device exchange (including the built-in microphone requirement and the picker), the open and the web + * denial overlay. + * @private + */ +const PERMISSION_TIMEOUT_MS = 60000; + +/** + * The getUserMedia timeout when the platform does not announce the preferred audio input device exchange: counted in + * wall-clock time, as before the exchange existed. + * @private + */ +const BASE_PERMISSION_TIMEOUT_MS = 45000; + /** * @typedef {Object} AirConsole~GetUserMediaConstraint - * @property {boolean} audio - Whether to request microphone permissions. Must be provided; video is not supported. + * @property {boolean|MediaTrackConstraints} audio - Whether to request microphone permissions, or the audio track + * constraints. Must be truthy; video is not supported. */ /** * Requests microphone permissions for the controller device. * Can only be called by a controller (not the screen). Video constraints are not supported. * - * @param {AirConsole~GetUserMediaConstraint} constraints - Media constraints. Must include `audio: true`. + * @param {AirConsole~GetUserMediaConstraint} constraints - Media constraints. Must include a truthy `audio`. * Video constraints are not supported and will cause the promise to reject with * {@link AirConsole.USER_MEDIA_ERROR_TYPE}.invalidConstraints. * @return {Promise} Resolves with the granted {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaStream MediaStream} @@ -827,6 +844,12 @@ const mediaPermissionCallbacks_ = new WeakMap(); * Note: callers are responsible for stopping stream tracks when the stream is no longer needed: * `stream.getTracks().forEach(function(t) { t.stop(); })` * + * On a platform that enables the preferred audio input exchange (e.g. a car head unit), the platform may prefer + * another audio input, such as the phone's built-in microphone; the API then asks the browser for that device, + * which opens another microphone if it is unavailable. There a request that stays unsettled for 60 s of unpaused + * time rejects with {@link AirConsole.USER_MEDIA_ERROR_TYPE}.timeout; the clock stops while the screen is paused. + * Everywhere else the timeout is 45 s. + * * @example * airconsole.getUserMedia({ audio: true }).then(function(stream) { * console.info('Media access granted', stream); @@ -867,10 +890,21 @@ AirConsole.prototype.getUserMedia = function getUserMedia(constraints) { return new Promise(function (resolve, reject) { me.mediaPermissionConstraints_ = constraints; me.mediaPermissionPending_ = true; + me.mediaPermissionRequestId_ = (me.mediaPermissionRequestId_ || 0) + 1; + me.mediaPermissionHandOffAccepted_ = false; + me.preferredAudioInputDeviceHandler_ = null; + me.mediaPermissionStreams_ = []; + // Taken per request, so a later ready cannot switch flows while a request is pending. + me.mediaPermissionUsesExchange_ = me.audioInputExchangeEnabled_ === true; mediaPermissionCallbacks_.set(me, { resolve: resolve, reject: reject }); - me.mediaPermissionTimeout_ = setTimeout(function() { - me.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); - }, 45000); + if (me.mediaPermissionUsesExchange_) { + me.mediaPermissionRemainingMs_ = PERMISSION_TIMEOUT_MS; + me.updatePermissionClock_(); + } else { + me.mediaPermissionTimeout_ = setTimeout(function() { + me.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); + }, BASE_PERMISSION_TIMEOUT_MS); + } // Send the request to the platform to decide where and how the user media request needs to take place based on // browser or controller environment. @@ -883,6 +917,11 @@ AirConsole.prototype.cleanUpMediaPermission_ = function cleanUpMediaPermission_( this.mediaPermissionPending_ = false; this.mediaPermissionConstraints_ = undefined; this.mediaPermissionTimeout_ = undefined; + this.mediaPermissionTimerStartedAt_ = undefined; + this.mediaPermissionHandOffAccepted_ = false; + this.preferredAudioInputDeviceHandler_ = null; + this.mediaPermissionStreams_ = []; + this.mediaPermissionUsesExchange_ = false; this.cachedMediaError_ = null; mediaPermissionCallbacks_.delete(this); } @@ -899,16 +938,399 @@ AirConsole.prototype.rejectMediaPermission_ = function rejectMediaPermission_(er if (cb) { cb.reject(error); } } +/** + * Returns true while the screen is paused, as published in its presence (`paused = { state, reason }`). + * The screen only publishes `paused` once its pause state first changes, so a missing `paused`, or a missing screen + * entry, means "not paused". + * @private + * @return {boolean} + */ +AirConsole.prototype.isScreenPaused_ = function isScreenPaused_() { + const screen = this.devices && this.devices[AirConsole.SCREEN]; + return !!(screen && screen.paused && screen.paused.state); +}; + +/** + * Starts or stops the permission clock to match the screen's pause state, keeping the remaining time across a pause. + * Called when a request starts, on `ready`, and whenever the screen's presence changes. Does nothing when no request + * is pending, or for a request without the exchange, whose timeout is the base one. + * @private + */ +AirConsole.prototype.updatePermissionClock_ = function updatePermissionClock_() { + var me = this; + if (!me.mediaPermissionPending_ || !me.mediaPermissionUsesExchange_) { + return; + } + const running = me.mediaPermissionTimeout_ !== undefined; + if (me.isScreenPaused_()) { + if (running) { + clearTimeout(me.mediaPermissionTimeout_); + me.mediaPermissionTimeout_ = undefined; + me.mediaPermissionRemainingMs_ -= Date.now() - me.mediaPermissionTimerStartedAt_; + me.mediaPermissionTimerStartedAt_ = undefined; + } + } else if (!running) { + me.mediaPermissionTimerStartedAt_ = Date.now(); + me.mediaPermissionTimeout_ = setTimeout(function () { + me.expireMediaPermission_(); + }, Math.max(0, me.mediaPermissionRemainingMs_)); + } +}; + +/** + * Ends the pending request when the permission timeout expires, or on destroy(): stops any stream the request holds, + * tells the platform, then rejects the game's promise. Sending before rejecting means the platform receives the + * failure before any retry the game sends. + * @private + */ +AirConsole.prototype.expireMediaPermission_ = function expireMediaPermission_() { + this.mediaPermissionTimeout_ = undefined; + this.stopMediaPermissionStreams_(); + this.sendEvent_('userMediaRequestFailed', { + reason: 'timeout', + error: AirConsole.USER_MEDIA_ERROR_TYPE.timeout, + }); + this.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); +}; + +/** + * Opens the stream on the permission hand-off, as before the preferred audio input device exchange existed. Used when + * the platform does not announce the exchange. + * @private + * @param {string} type - 'userMediaPermissionGranted' or 'promptUserMediaPermission' + */ +AirConsole.prototype.startBaseMediaPermissionRequest_ = function startBaseMediaPermissionRequest_(type) { + var me = this; + navigator.mediaDevices.getUserMedia(me.mediaPermissionConstraints_).then( + function success(stream) { + if (!me.mediaPermissionPending_) { + // The request ended (timeout or destroy()) while the browser permission dialog was still open; + // the outer Promise is already settled — stop the orphaned stream to release hardware. + stream.getTracks().forEach(function (t) { t.stop(); }); + return; + } + // Note: 'userMediaPermissionGranted' is both sent upward (controller → platform) and + // received downward (platform → controller for native controllers). The direction is + // determined by context: outbound is sent here; inbound is handled by the event branch. + me.sendEvent_('userMediaPermissionGranted', { + constraints: me.mediaPermissionConstraints_, + }); + me.resolveMediaPermission_(stream); + }, + function failure(error) { + if (!me.mediaPermissionPending_) { + return; + } + // Native controller: platform already granted permission but stream open failed. + if (type === 'userMediaPermissionGranted') { + me.rejectMediaPermission_(error); + } else { + me.cachedMediaError_ = error; + me.sendEvent_('userMediaPermissionDenied'); + } + }); +}; + +/** + * Returns true if requestId is the request that is still pending. + * @private + * @param {number} requestId + * @return {boolean} + */ +AirConsole.prototype.isCurrentMediaPermission_ = function isCurrentMediaPermission_(requestId) { + return this.mediaPermissionPending_ && this.mediaPermissionRequestId_ === requestId; +}; + +/** + * Stops the tracks of every stream the pending request has opened and not handed to the game. + * @private + */ +AirConsole.prototype.stopMediaPermissionStreams_ = function stopMediaPermissionStreams_() { + (this.mediaPermissionStreams_ || []).forEach(function (stream) { + stream.getTracks().forEach(function (t) { t.stop(); }); + }); + this.mediaPermissionStreams_ = []; +}; + +/** + * Returns the name the platform receives for an error: the message of an AirConsoleUserMediaError, otherwise the + * error's name (e.g. a DOMException's), or String(error) if it has none. + * @private + * @param {*} error + * @return {string} + */ +AirConsole.getUserMediaErrorName_ = function getUserMediaErrorName_(error) { + if (error instanceof AirConsoleUserMediaError) { + return error.message; + } + return (error && error.name) || String(error); +}; + +/** + * Returns the device id the stream's audio track captures from, or '' when the stream has no live audio track or + * the id is unavailable. + * @private + * @param {MediaStream} stream + * @return {string} + */ +AirConsole.getActiveAudioInputDeviceId_ = function getActiveAudioInputDeviceId_(stream) { + const track = stream && stream.getAudioTracks && stream.getAudioTracks()[0]; + if (!track || track.readyState === 'ended' || !track.getSettings) { + return ''; + } + return track.getSettings().deviceId || ''; +}; + +/** + * Returns the audio inputs from enumerateDevices(), or an empty list if they cannot be enumerated. + * @private + * @return {Promise>} + */ +AirConsole.enumerateAudioInputDevices_ = function enumerateAudioInputDevices_() { + if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) { + return Promise.resolve([]); + } + return navigator.mediaDevices.enumerateDevices().then(function (devices) { + return devices.filter(function (device) { + return device.kind === 'audioinput'; + }).map(function (device) { + return { deviceId: device.deviceId, label: device.label }; + }); + }, function () { + return []; + }); +}; + +/** + * Returns the game's constraints with deviceId merged into `audio` as a plain (non-exact) value, so the browser + * treats it as a preference and opens another microphone when that device is gone. Returns the constraints unchanged + * when deviceId is null. + * @private + * @param {AirConsole~GetUserMediaConstraint} constraints + * @param {string|null} deviceId + * @return {Object} + */ +AirConsole.getConstraintsWithAudioInputDevice_ = function getConstraintsWithAudioInputDevice_(constraints, deviceId) { + if (!deviceId) { + return constraints; + } + const audio = typeof constraints.audio === 'object' ? Object.assign({}, constraints.audio) : {}; + audio.deviceId = deviceId; + return Object.assign({}, constraints, { audio: audio }); +}; + +/** + * Asks the platform which audio input the stream should use and waits for its preferredAudioInputDevice reply. + * The reply is accepted once per request, and only while it is awaited. An accepted reply marks the exchange + * completed, so later requests use the later-request flow. + * @private + * @param {number} requestId + * @param {Object} payload - { devices, activeDeviceId } on a first request; empty on a later one. + * @param {function(string|null)} onReply + */ +AirConsole.prototype.requestPreferredAudioInputDevice_ = function requestPreferredAudioInputDevice_(requestId, + payload, onReply) { + var me = this; + if (!me.isCurrentMediaPermission_(requestId)) { + return; + } + me.preferredAudioInputDeviceHandler_ = function (deviceId) { + me.preferredAudioInputDeviceHandler_ = null; + me.audioInputExchangeCompleted_ = true; + onReply(deviceId || null); + }; + me.sendEvent_('requestPreferredAudioInputDevice', payload); +}; + +/** + * Sends userMediaPermissionGranted, hands the stream to the game, then reports the audio inputs and the device the + * stream actually captures from (audioInputDevicesReported). Stops the stream instead if the request has ended. + * @private + * @param {number} requestId + * @param {MediaStream} stream + */ +AirConsole.prototype.resolveMediaPermissionWithReport_ = function resolveMediaPermissionWithReport_(requestId, + stream) { + var me = this; + if (!me.isCurrentMediaPermission_(requestId)) { + stream.getTracks().forEach(function (t) { t.stop(); }); + return; + } + me.mediaPermissionStreams_ = me.mediaPermissionStreams_.filter(function (s) { return s !== stream; }); + // Note: 'userMediaPermissionGranted' is both sent upward (controller → platform) and + // received downward (platform → controller for native controllers). The direction is + // determined by context: outbound is sent here; inbound is handled by the event branch. + me.sendEvent_('userMediaPermissionGranted', { + constraints: me.mediaPermissionConstraints_, + }); + me.resolveMediaPermission_(stream); + AirConsole.enumerateAudioInputDevices_().then(function (devices) { + me.sendEvent_('audioInputDevicesReported', { + devices: devices, + activeDeviceId: AirConsole.getActiveAudioInputDeviceId_(stream), + }); + }); +}; + +/** + * Ends the pending request after an open failed in a way the denial path does not handle (any error after the + * native app's grant, a NotAllowedError included, or any other than NotAllowedError on web): sends + * userMediaRequestFailed with reason 'open-error', then rejects with the error. + * @private + * @param {number} requestId + * @param {*} error + */ +AirConsole.prototype.failMediaPermissionOpen_ = function failMediaPermissionOpen_(requestId, error) { + if (!this.isCurrentMediaPermission_(requestId)) { + return; + } + this.sendEvent_('userMediaRequestFailed', { + reason: 'open-error', + error: AirConsole.getUserMediaErrorName_(error), + }); + this.rejectMediaPermission_(error); +}; + +/** + * Ends the pending request after an open that followed stream #1 failed: a fresh open or stream #2 on a first + * request, or a later request's single open. On web a NotAllowedError is a permission denial and takes the base's + * denial path. Any other error, and every error in the app, where the native app has just granted the permission, + * reaches the game through failMediaPermissionOpen_. + * @private + * @param {number} requestId + * @param {string} handOffType - 'promptUserMediaPermission' (web) or 'userMediaPermissionGranted' (native app). + * @param {*} error + */ +AirConsole.prototype.failMediaPermissionReopen_ = function failMediaPermissionReopen_(requestId, handOffType, + error) { + if (!this.isCurrentMediaPermission_(requestId)) { + return; + } + if (handOffType === 'promptUserMediaPermission' && error && error.name === 'NotAllowedError') { + this.cachedMediaError_ = error; + this.sendEvent_('userMediaPermissionDenied'); + } else { + this.failMediaPermissionOpen_(requestId, error); + } +}; + +/** + * Opens a stream for the pending request, holding it until it is handed to the game. + * @private + * @param {number} requestId + * @param {Object} constraints + * @return {Promise} null when the request ended while the browser was opening. + */ +AirConsole.prototype.openMediaPermissionStream_ = function openMediaPermissionStream_(requestId, constraints) { + var me = this; + return navigator.mediaDevices.getUserMedia(constraints).then(function (stream) { + if (!me.isCurrentMediaPermission_(requestId)) { + // The request ended while the browser was opening; stop the orphaned stream to release hardware. + stream.getTracks().forEach(function (t) { t.stop(); }); + return null; + } + me.mediaPermissionStreams_.push(stream); + return stream; + }); +}; + +/** + * Used until the platform has answered once on this instance (a game's first request, or the next one after a + * request that ended before the reply): opens stream #1 with the game's constraints, asks for the preferred audio + * input device with the device list, then keeps stream #1, reopens a fresh stream if it ended, or switches to + * stream #2. + * @private + * @param {number} requestId + * @param {string} handOffType - 'promptUserMediaPermission' (web) or 'userMediaPermissionGranted' (native app). + */ +AirConsole.prototype.startFirstMediaPermissionRequest_ = function startFirstMediaPermissionRequest_(requestId, + handOffType) { + var me = this; + const constraints = me.mediaPermissionConstraints_; + me.openMediaPermissionStream_(requestId, constraints).then(function (stream1) { + if (!stream1) { + return; + } + AirConsole.enumerateAudioInputDevices_().then(function (devices) { + me.requestPreferredAudioInputDevice_(requestId, { + devices: devices, + activeDeviceId: AirConsole.getActiveAudioInputDeviceId_(stream1), + }, function (deviceId) { + const track = stream1.getAudioTracks()[0]; + const live = !!track && track.readyState === 'live'; + if (live && (deviceId === null || deviceId === AirConsole.getActiveAudioInputDeviceId_(stream1))) { + me.resolveMediaPermissionWithReport_(requestId, stream1); + return; + } + // Stream #1 ended, or is on another device: iOS refuses a second audio input while the first is live, + // so stop it before opening the next one. + me.stopMediaPermissionStreams_(); + me.openMediaPermissionStream_(requestId, + AirConsole.getConstraintsWithAudioInputDevice_(constraints, deviceId)).then(function (stream) { + if (stream) { + me.resolveMediaPermissionWithReport_(requestId, stream); + } + }, function (error) { + me.failMediaPermissionReopen_(requestId, handOffType, error); + }); + }); + }); + }, function (error) { + if (!me.isCurrentMediaPermission_(requestId)) { + return; + } + if (handOffType === 'userMediaPermissionGranted') { + // Native controller: platform already granted permission but stream open failed. + me.failMediaPermissionOpen_(requestId, error); + } else { + me.cachedMediaError_ = error; + me.sendEvent_('userMediaPermissionDenied'); + } + }); +}; + +/** + * Used once the platform has answered a previous request: asks for the preferred audio input device before opening + * anything, then opens once, with the preferred device merged in as a non-exact constraint when the reply names one. + * @private + * @param {number} requestId + * @param {string} handOffType - 'promptUserMediaPermission' (web) or 'userMediaPermissionGranted' (native app). + */ +AirConsole.prototype.startLaterMediaPermissionRequest_ = function startLaterMediaPermissionRequest_(requestId, + handOffType) { + var me = this; + const constraints = me.mediaPermissionConstraints_; + me.requestPreferredAudioInputDevice_(requestId, {}, function (deviceId) { + me.openMediaPermissionStream_(requestId, + AirConsole.getConstraintsWithAudioInputDevice_(constraints, deviceId)).then(function (stream) { + if (stream) { + me.resolveMediaPermissionWithReport_(requestId, stream); + } + }, function (error) { + // On web a revoked permission is first seen here, as a NotAllowedError. + me.failMediaPermissionReopen_(requestId, handOffType, error); + }); + }); +}; + /** * Releases resources held by this AirConsole instance. * Call this when the instance is no longer needed — for example, when navigating * away in a single-page application — to remove the message event listener and - * cancel any pending getUserMedia timeout. Failing to call destroy() in such - * environments will leave a dangling listener and possibly a live timeout. + * end any pending getUserMedia request: it is rejected with + * {@link AirConsole.USER_MEDIA_ERROR_TYPE}.timeout, and with the preferred audio input + * exchange its streams are stopped and userMediaRequestFailed is sent first. + * Failing to call destroy() in such environments will leave a dangling listener and + * possibly a live timeout. */ AirConsole.prototype.destroy = function destroy() { window.removeEventListener('message', this.messageEventListener_); - if (this.mediaPermissionPending_) { + if (this.mediaPermissionPending_ && this.mediaPermissionUsesExchange_) { + // End the request like a timeout: stop any stream it holds (stream #1 while the reply is awaited), tell the + // platform so it closes its prompts, then reject. The timer is cleared first, so it cannot expire a second time. + clearTimeout(this.mediaPermissionTimeout_); + this.expireMediaPermission_(); + } else if (this.mediaPermissionPending_) { this.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); } else if (this.mediaPermissionTimeout_) { clearTimeout(this.mediaPermissionTimeout_); @@ -1557,6 +1979,9 @@ AirConsole.prototype.onPostMessage_ = function(event) { var sender = data.device_id; me.devices[sender] = data.device_data; + if (sender === AirConsole.SCREEN) { + me.updatePermissionClock_(); + } me.onDeviceStateChange(sender, data.device_data); var is_connect = me.isLocationLoadedMessage_(game_url_before, game_url, game_url_after); var is_disconnect = me.isLocationUnloadedMessage_(game_url_before, game_url, game_url_after); @@ -1596,6 +2021,8 @@ AirConsole.prototype.onPostMessage_ = function(event) { } else if (data.action === "ready") { me.device_id = data.device_id; me.devices = data.devices; + me.audioInputExchangeEnabled_ = data.audioInputExchangeEnabled === true; + me.updatePermissionClock_(); if (me.server_time_offset !== false) { me.server_time_offset = data.server_time_offset || 0; } @@ -1694,7 +2121,7 @@ AirConsole.prototype.onPostMessage_ = function(event) { const { type } = data; // Guard: ignore stale platform messages that arrive after state has been cleaned up - // (e.g. after the 30-second timeout has already resolved the pending Promise). + // (e.g. after the timeout has already rejected the pending Promise). if (!me.mediaPermissionPending_) { return; } @@ -1711,35 +2138,26 @@ AirConsole.prototype.onPostMessage_ = function(event) { me.rejectMediaPermission_(error); } } else if (type === 'userMediaPermissionGranted' || type === 'promptUserMediaPermission') { - navigator.mediaDevices.getUserMedia(me.mediaPermissionConstraints_).then( - function success(stream) { - if (!me.mediaPermissionPending_) { - // Timeout fired while the browser permission dialog was still open; - // the outer Promise is already settled — stop the orphaned stream to release hardware. - stream.getTracks().forEach(function (t) { t.stop(); }); - return; - } - // Note: 'userMediaPermissionGranted' is both sent upward (controller → platform) and - // received downward (platform → controller for native controllers). The direction is - // determined by context: outbound is sent here; inbound is handled by this event branch. - me.sendEvent_('userMediaPermissionGranted', { - constraints: me.mediaPermissionConstraints_, - }); - me.resolveMediaPermission_(stream); - }, - function failure(error) { - if (!me.mediaPermissionPending_) { - return; - } - // Native controller: platform already granted permission but stream open failed. - if (type === 'userMediaPermissionGranted') { - me.rejectMediaPermission_(error); - } else { - me.cachedMediaError_ = error; - me.sendEvent_('userMediaPermissionDenied'); - } - } - ); + if (!me.mediaPermissionUsesExchange_) { + me.startBaseMediaPermissionRequest_(type); + return; + } + // The hand-off is accepted once per request, so a duplicate hand-off cannot open a second stream. + if (me.mediaPermissionHandOffAccepted_) { + return; + } + me.mediaPermissionHandOffAccepted_ = true; + if (me.audioInputExchangeCompleted_) { + me.startLaterMediaPermissionRequest_(me.mediaPermissionRequestId_, type); + } else { + me.startFirstMediaPermissionRequest_(me.mediaPermissionRequestId_, type); + } + } else if (type === 'preferredAudioInputDevice') { + // Accepted only while awaited. A reply meant for an earlier request is ignored: the platform's messages arrive + // in order, so it arrives before this request's hand-off, while no reply is awaited. + if (me.preferredAudioInputDeviceHandler_) { + me.preferredAudioInputDeviceHandler_(data.data && data.data.deviceId); + } } } }; diff --git a/beta/airconsole-1.12.0.js b/beta/airconsole-1.12.0.js index 8e67552..9b4c8b4 100644 --- a/beta/airconsole-1.12.0.js +++ b/beta/airconsole-1.12.0.js @@ -806,16 +806,33 @@ AirConsole.prototype.onUserMediaAccessDenied = function (device_id) {}; */ const mediaPermissionCallbacks_ = new WeakMap(); +/** + * How long a getUserMedia request may stay unsettled, counted in unpaused time: the clock stops while the screen is + * paused. It covers everything a request waits on: the car-connection notice, the permission flow, the preferred + * audio input device exchange (including the built-in microphone requirement and the picker), the open and the web + * denial overlay. + * @private + */ +const PERMISSION_TIMEOUT_MS = 60000; + +/** + * The getUserMedia timeout when the platform does not announce the preferred audio input device exchange: counted in + * wall-clock time, as before the exchange existed. + * @private + */ +const BASE_PERMISSION_TIMEOUT_MS = 45000; + /** * @typedef {Object} AirConsole~GetUserMediaConstraint - * @property {boolean} audio - Whether to request microphone permissions. Must be provided; video is not supported. + * @property {boolean|MediaTrackConstraints} audio - Whether to request microphone permissions, or the audio track + * constraints. Must be truthy; video is not supported. */ /** * Requests microphone permissions for the controller device. * Can only be called by a controller (not the screen). Video constraints are not supported. * - * @param {AirConsole~GetUserMediaConstraint} constraints - Media constraints. Must include `audio: true`. + * @param {AirConsole~GetUserMediaConstraint} constraints - Media constraints. Must include a truthy `audio`. * Video constraints are not supported and will cause the promise to reject with * {@link AirConsole.USER_MEDIA_ERROR_TYPE}.invalidConstraints. * @return {Promise} Resolves with the granted {@link https://developer.mozilla.org/en-US/docs/Web/API/MediaStream MediaStream} @@ -827,6 +844,12 @@ const mediaPermissionCallbacks_ = new WeakMap(); * Note: callers are responsible for stopping stream tracks when the stream is no longer needed: * `stream.getTracks().forEach(function(t) { t.stop(); })` * + * On a platform that enables the preferred audio input exchange (e.g. a car head unit), the platform may prefer + * another audio input, such as the phone's built-in microphone; the API then asks the browser for that device, + * which opens another microphone if it is unavailable. There a request that stays unsettled for 60 s of unpaused + * time rejects with {@link AirConsole.USER_MEDIA_ERROR_TYPE}.timeout; the clock stops while the screen is paused. + * Everywhere else the timeout is 45 s. + * * @example * airconsole.getUserMedia({ audio: true }).then(function(stream) { * console.info('Media access granted', stream); @@ -867,10 +890,21 @@ AirConsole.prototype.getUserMedia = function getUserMedia(constraints) { return new Promise(function (resolve, reject) { me.mediaPermissionConstraints_ = constraints; me.mediaPermissionPending_ = true; + me.mediaPermissionRequestId_ = (me.mediaPermissionRequestId_ || 0) + 1; + me.mediaPermissionHandOffAccepted_ = false; + me.preferredAudioInputDeviceHandler_ = null; + me.mediaPermissionStreams_ = []; + // Taken per request, so a later ready cannot switch flows while a request is pending. + me.mediaPermissionUsesExchange_ = me.audioInputExchangeEnabled_ === true; mediaPermissionCallbacks_.set(me, { resolve: resolve, reject: reject }); - me.mediaPermissionTimeout_ = setTimeout(function() { - me.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); - }, 45000); + if (me.mediaPermissionUsesExchange_) { + me.mediaPermissionRemainingMs_ = PERMISSION_TIMEOUT_MS; + me.updatePermissionClock_(); + } else { + me.mediaPermissionTimeout_ = setTimeout(function() { + me.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); + }, BASE_PERMISSION_TIMEOUT_MS); + } // Send the request to the platform to decide where and how the user media request needs to take place based on // browser or controller environment. @@ -883,6 +917,11 @@ AirConsole.prototype.cleanUpMediaPermission_ = function cleanUpMediaPermission_( this.mediaPermissionPending_ = false; this.mediaPermissionConstraints_ = undefined; this.mediaPermissionTimeout_ = undefined; + this.mediaPermissionTimerStartedAt_ = undefined; + this.mediaPermissionHandOffAccepted_ = false; + this.preferredAudioInputDeviceHandler_ = null; + this.mediaPermissionStreams_ = []; + this.mediaPermissionUsesExchange_ = false; this.cachedMediaError_ = null; mediaPermissionCallbacks_.delete(this); } @@ -899,16 +938,399 @@ AirConsole.prototype.rejectMediaPermission_ = function rejectMediaPermission_(er if (cb) { cb.reject(error); } } +/** + * Returns true while the screen is paused, as published in its presence (`paused = { state, reason }`). + * The screen only publishes `paused` once its pause state first changes, so a missing `paused`, or a missing screen + * entry, means "not paused". + * @private + * @return {boolean} + */ +AirConsole.prototype.isScreenPaused_ = function isScreenPaused_() { + const screen = this.devices && this.devices[AirConsole.SCREEN]; + return !!(screen && screen.paused && screen.paused.state); +}; + +/** + * Starts or stops the permission clock to match the screen's pause state, keeping the remaining time across a pause. + * Called when a request starts, on `ready`, and whenever the screen's presence changes. Does nothing when no request + * is pending, or for a request without the exchange, whose timeout is the base one. + * @private + */ +AirConsole.prototype.updatePermissionClock_ = function updatePermissionClock_() { + var me = this; + if (!me.mediaPermissionPending_ || !me.mediaPermissionUsesExchange_) { + return; + } + const running = me.mediaPermissionTimeout_ !== undefined; + if (me.isScreenPaused_()) { + if (running) { + clearTimeout(me.mediaPermissionTimeout_); + me.mediaPermissionTimeout_ = undefined; + me.mediaPermissionRemainingMs_ -= Date.now() - me.mediaPermissionTimerStartedAt_; + me.mediaPermissionTimerStartedAt_ = undefined; + } + } else if (!running) { + me.mediaPermissionTimerStartedAt_ = Date.now(); + me.mediaPermissionTimeout_ = setTimeout(function () { + me.expireMediaPermission_(); + }, Math.max(0, me.mediaPermissionRemainingMs_)); + } +}; + +/** + * Ends the pending request when the permission timeout expires, or on destroy(): stops any stream the request holds, + * tells the platform, then rejects the game's promise. Sending before rejecting means the platform receives the + * failure before any retry the game sends. + * @private + */ +AirConsole.prototype.expireMediaPermission_ = function expireMediaPermission_() { + this.mediaPermissionTimeout_ = undefined; + this.stopMediaPermissionStreams_(); + this.sendEvent_('userMediaRequestFailed', { + reason: 'timeout', + error: AirConsole.USER_MEDIA_ERROR_TYPE.timeout, + }); + this.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); +}; + +/** + * Opens the stream on the permission hand-off, as before the preferred audio input device exchange existed. Used when + * the platform does not announce the exchange. + * @private + * @param {string} type - 'userMediaPermissionGranted' or 'promptUserMediaPermission' + */ +AirConsole.prototype.startBaseMediaPermissionRequest_ = function startBaseMediaPermissionRequest_(type) { + var me = this; + navigator.mediaDevices.getUserMedia(me.mediaPermissionConstraints_).then( + function success(stream) { + if (!me.mediaPermissionPending_) { + // The request ended (timeout or destroy()) while the browser permission dialog was still open; + // the outer Promise is already settled — stop the orphaned stream to release hardware. + stream.getTracks().forEach(function (t) { t.stop(); }); + return; + } + // Note: 'userMediaPermissionGranted' is both sent upward (controller → platform) and + // received downward (platform → controller for native controllers). The direction is + // determined by context: outbound is sent here; inbound is handled by the event branch. + me.sendEvent_('userMediaPermissionGranted', { + constraints: me.mediaPermissionConstraints_, + }); + me.resolveMediaPermission_(stream); + }, + function failure(error) { + if (!me.mediaPermissionPending_) { + return; + } + // Native controller: platform already granted permission but stream open failed. + if (type === 'userMediaPermissionGranted') { + me.rejectMediaPermission_(error); + } else { + me.cachedMediaError_ = error; + me.sendEvent_('userMediaPermissionDenied'); + } + }); +}; + +/** + * Returns true if requestId is the request that is still pending. + * @private + * @param {number} requestId + * @return {boolean} + */ +AirConsole.prototype.isCurrentMediaPermission_ = function isCurrentMediaPermission_(requestId) { + return this.mediaPermissionPending_ && this.mediaPermissionRequestId_ === requestId; +}; + +/** + * Stops the tracks of every stream the pending request has opened and not handed to the game. + * @private + */ +AirConsole.prototype.stopMediaPermissionStreams_ = function stopMediaPermissionStreams_() { + (this.mediaPermissionStreams_ || []).forEach(function (stream) { + stream.getTracks().forEach(function (t) { t.stop(); }); + }); + this.mediaPermissionStreams_ = []; +}; + +/** + * Returns the name the platform receives for an error: the message of an AirConsoleUserMediaError, otherwise the + * error's name (e.g. a DOMException's), or String(error) if it has none. + * @private + * @param {*} error + * @return {string} + */ +AirConsole.getUserMediaErrorName_ = function getUserMediaErrorName_(error) { + if (error instanceof AirConsoleUserMediaError) { + return error.message; + } + return (error && error.name) || String(error); +}; + +/** + * Returns the device id the stream's audio track captures from, or '' when the stream has no live audio track or + * the id is unavailable. + * @private + * @param {MediaStream} stream + * @return {string} + */ +AirConsole.getActiveAudioInputDeviceId_ = function getActiveAudioInputDeviceId_(stream) { + const track = stream && stream.getAudioTracks && stream.getAudioTracks()[0]; + if (!track || track.readyState === 'ended' || !track.getSettings) { + return ''; + } + return track.getSettings().deviceId || ''; +}; + +/** + * Returns the audio inputs from enumerateDevices(), or an empty list if they cannot be enumerated. + * @private + * @return {Promise>} + */ +AirConsole.enumerateAudioInputDevices_ = function enumerateAudioInputDevices_() { + if (!navigator.mediaDevices || !navigator.mediaDevices.enumerateDevices) { + return Promise.resolve([]); + } + return navigator.mediaDevices.enumerateDevices().then(function (devices) { + return devices.filter(function (device) { + return device.kind === 'audioinput'; + }).map(function (device) { + return { deviceId: device.deviceId, label: device.label }; + }); + }, function () { + return []; + }); +}; + +/** + * Returns the game's constraints with deviceId merged into `audio` as a plain (non-exact) value, so the browser + * treats it as a preference and opens another microphone when that device is gone. Returns the constraints unchanged + * when deviceId is null. + * @private + * @param {AirConsole~GetUserMediaConstraint} constraints + * @param {string|null} deviceId + * @return {Object} + */ +AirConsole.getConstraintsWithAudioInputDevice_ = function getConstraintsWithAudioInputDevice_(constraints, deviceId) { + if (!deviceId) { + return constraints; + } + const audio = typeof constraints.audio === 'object' ? Object.assign({}, constraints.audio) : {}; + audio.deviceId = deviceId; + return Object.assign({}, constraints, { audio: audio }); +}; + +/** + * Asks the platform which audio input the stream should use and waits for its preferredAudioInputDevice reply. + * The reply is accepted once per request, and only while it is awaited. An accepted reply marks the exchange + * completed, so later requests use the later-request flow. + * @private + * @param {number} requestId + * @param {Object} payload - { devices, activeDeviceId } on a first request; empty on a later one. + * @param {function(string|null)} onReply + */ +AirConsole.prototype.requestPreferredAudioInputDevice_ = function requestPreferredAudioInputDevice_(requestId, + payload, onReply) { + var me = this; + if (!me.isCurrentMediaPermission_(requestId)) { + return; + } + me.preferredAudioInputDeviceHandler_ = function (deviceId) { + me.preferredAudioInputDeviceHandler_ = null; + me.audioInputExchangeCompleted_ = true; + onReply(deviceId || null); + }; + me.sendEvent_('requestPreferredAudioInputDevice', payload); +}; + +/** + * Sends userMediaPermissionGranted, hands the stream to the game, then reports the audio inputs and the device the + * stream actually captures from (audioInputDevicesReported). Stops the stream instead if the request has ended. + * @private + * @param {number} requestId + * @param {MediaStream} stream + */ +AirConsole.prototype.resolveMediaPermissionWithReport_ = function resolveMediaPermissionWithReport_(requestId, + stream) { + var me = this; + if (!me.isCurrentMediaPermission_(requestId)) { + stream.getTracks().forEach(function (t) { t.stop(); }); + return; + } + me.mediaPermissionStreams_ = me.mediaPermissionStreams_.filter(function (s) { return s !== stream; }); + // Note: 'userMediaPermissionGranted' is both sent upward (controller → platform) and + // received downward (platform → controller for native controllers). The direction is + // determined by context: outbound is sent here; inbound is handled by the event branch. + me.sendEvent_('userMediaPermissionGranted', { + constraints: me.mediaPermissionConstraints_, + }); + me.resolveMediaPermission_(stream); + AirConsole.enumerateAudioInputDevices_().then(function (devices) { + me.sendEvent_('audioInputDevicesReported', { + devices: devices, + activeDeviceId: AirConsole.getActiveAudioInputDeviceId_(stream), + }); + }); +}; + +/** + * Ends the pending request after an open failed in a way the denial path does not handle (any error after the + * native app's grant, a NotAllowedError included, or any other than NotAllowedError on web): sends + * userMediaRequestFailed with reason 'open-error', then rejects with the error. + * @private + * @param {number} requestId + * @param {*} error + */ +AirConsole.prototype.failMediaPermissionOpen_ = function failMediaPermissionOpen_(requestId, error) { + if (!this.isCurrentMediaPermission_(requestId)) { + return; + } + this.sendEvent_('userMediaRequestFailed', { + reason: 'open-error', + error: AirConsole.getUserMediaErrorName_(error), + }); + this.rejectMediaPermission_(error); +}; + +/** + * Ends the pending request after an open that followed stream #1 failed: a fresh open or stream #2 on a first + * request, or a later request's single open. On web a NotAllowedError is a permission denial and takes the base's + * denial path. Any other error, and every error in the app, where the native app has just granted the permission, + * reaches the game through failMediaPermissionOpen_. + * @private + * @param {number} requestId + * @param {string} handOffType - 'promptUserMediaPermission' (web) or 'userMediaPermissionGranted' (native app). + * @param {*} error + */ +AirConsole.prototype.failMediaPermissionReopen_ = function failMediaPermissionReopen_(requestId, handOffType, + error) { + if (!this.isCurrentMediaPermission_(requestId)) { + return; + } + if (handOffType === 'promptUserMediaPermission' && error && error.name === 'NotAllowedError') { + this.cachedMediaError_ = error; + this.sendEvent_('userMediaPermissionDenied'); + } else { + this.failMediaPermissionOpen_(requestId, error); + } +}; + +/** + * Opens a stream for the pending request, holding it until it is handed to the game. + * @private + * @param {number} requestId + * @param {Object} constraints + * @return {Promise} null when the request ended while the browser was opening. + */ +AirConsole.prototype.openMediaPermissionStream_ = function openMediaPermissionStream_(requestId, constraints) { + var me = this; + return navigator.mediaDevices.getUserMedia(constraints).then(function (stream) { + if (!me.isCurrentMediaPermission_(requestId)) { + // The request ended while the browser was opening; stop the orphaned stream to release hardware. + stream.getTracks().forEach(function (t) { t.stop(); }); + return null; + } + me.mediaPermissionStreams_.push(stream); + return stream; + }); +}; + +/** + * Used until the platform has answered once on this instance (a game's first request, or the next one after a + * request that ended before the reply): opens stream #1 with the game's constraints, asks for the preferred audio + * input device with the device list, then keeps stream #1, reopens a fresh stream if it ended, or switches to + * stream #2. + * @private + * @param {number} requestId + * @param {string} handOffType - 'promptUserMediaPermission' (web) or 'userMediaPermissionGranted' (native app). + */ +AirConsole.prototype.startFirstMediaPermissionRequest_ = function startFirstMediaPermissionRequest_(requestId, + handOffType) { + var me = this; + const constraints = me.mediaPermissionConstraints_; + me.openMediaPermissionStream_(requestId, constraints).then(function (stream1) { + if (!stream1) { + return; + } + AirConsole.enumerateAudioInputDevices_().then(function (devices) { + me.requestPreferredAudioInputDevice_(requestId, { + devices: devices, + activeDeviceId: AirConsole.getActiveAudioInputDeviceId_(stream1), + }, function (deviceId) { + const track = stream1.getAudioTracks()[0]; + const live = !!track && track.readyState === 'live'; + if (live && (deviceId === null || deviceId === AirConsole.getActiveAudioInputDeviceId_(stream1))) { + me.resolveMediaPermissionWithReport_(requestId, stream1); + return; + } + // Stream #1 ended, or is on another device: iOS refuses a second audio input while the first is live, + // so stop it before opening the next one. + me.stopMediaPermissionStreams_(); + me.openMediaPermissionStream_(requestId, + AirConsole.getConstraintsWithAudioInputDevice_(constraints, deviceId)).then(function (stream) { + if (stream) { + me.resolveMediaPermissionWithReport_(requestId, stream); + } + }, function (error) { + me.failMediaPermissionReopen_(requestId, handOffType, error); + }); + }); + }); + }, function (error) { + if (!me.isCurrentMediaPermission_(requestId)) { + return; + } + if (handOffType === 'userMediaPermissionGranted') { + // Native controller: platform already granted permission but stream open failed. + me.failMediaPermissionOpen_(requestId, error); + } else { + me.cachedMediaError_ = error; + me.sendEvent_('userMediaPermissionDenied'); + } + }); +}; + +/** + * Used once the platform has answered a previous request: asks for the preferred audio input device before opening + * anything, then opens once, with the preferred device merged in as a non-exact constraint when the reply names one. + * @private + * @param {number} requestId + * @param {string} handOffType - 'promptUserMediaPermission' (web) or 'userMediaPermissionGranted' (native app). + */ +AirConsole.prototype.startLaterMediaPermissionRequest_ = function startLaterMediaPermissionRequest_(requestId, + handOffType) { + var me = this; + const constraints = me.mediaPermissionConstraints_; + me.requestPreferredAudioInputDevice_(requestId, {}, function (deviceId) { + me.openMediaPermissionStream_(requestId, + AirConsole.getConstraintsWithAudioInputDevice_(constraints, deviceId)).then(function (stream) { + if (stream) { + me.resolveMediaPermissionWithReport_(requestId, stream); + } + }, function (error) { + // On web a revoked permission is first seen here, as a NotAllowedError. + me.failMediaPermissionReopen_(requestId, handOffType, error); + }); + }); +}; + /** * Releases resources held by this AirConsole instance. * Call this when the instance is no longer needed — for example, when navigating * away in a single-page application — to remove the message event listener and - * cancel any pending getUserMedia timeout. Failing to call destroy() in such - * environments will leave a dangling listener and possibly a live timeout. + * end any pending getUserMedia request: it is rejected with + * {@link AirConsole.USER_MEDIA_ERROR_TYPE}.timeout, and with the preferred audio input + * exchange its streams are stopped and userMediaRequestFailed is sent first. + * Failing to call destroy() in such environments will leave a dangling listener and + * possibly a live timeout. */ AirConsole.prototype.destroy = function destroy() { window.removeEventListener('message', this.messageEventListener_); - if (this.mediaPermissionPending_) { + if (this.mediaPermissionPending_ && this.mediaPermissionUsesExchange_) { + // End the request like a timeout: stop any stream it holds (stream #1 while the reply is awaited), tell the + // platform so it closes its prompts, then reject. The timer is cleared first, so it cannot expire a second time. + clearTimeout(this.mediaPermissionTimeout_); + this.expireMediaPermission_(); + } else if (this.mediaPermissionPending_) { this.rejectMediaPermission_(new AirConsoleUserMediaError(AirConsole.USER_MEDIA_ERROR_TYPE.timeout)); } else if (this.mediaPermissionTimeout_) { clearTimeout(this.mediaPermissionTimeout_); @@ -1631,6 +2053,9 @@ AirConsole.prototype.onPostMessage_ = function(event) { var sender = data.device_id; me.devices[sender] = data.device_data; + if (sender === AirConsole.SCREEN) { + me.updatePermissionClock_(); + } me.onDeviceStateChange(sender, data.device_data); var is_connect = me.isLocationLoadedMessage_(game_url_before, game_url, game_url_after); var is_disconnect = me.isLocationUnloadedMessage_(game_url_before, game_url, game_url_after); @@ -1670,6 +2095,8 @@ AirConsole.prototype.onPostMessage_ = function(event) { } else if (data.action === "ready") { me.device_id = data.device_id; me.devices = data.devices; + me.audioInputExchangeEnabled_ = data.audioInputExchangeEnabled === true; + me.updatePermissionClock_(); if (me.server_time_offset !== false) { me.server_time_offset = data.server_time_offset || 0; } @@ -1770,7 +2197,7 @@ AirConsole.prototype.onPostMessage_ = function(event) { const { type } = data; // Guard: ignore stale platform messages that arrive after state has been cleaned up - // (e.g. after the 30-second timeout has already resolved the pending Promise). + // (e.g. after the timeout has already rejected the pending Promise). if (!me.mediaPermissionPending_) { return; } @@ -1787,35 +2214,26 @@ AirConsole.prototype.onPostMessage_ = function(event) { me.rejectMediaPermission_(error); } } else if (type === 'userMediaPermissionGranted' || type === 'promptUserMediaPermission') { - navigator.mediaDevices.getUserMedia(me.mediaPermissionConstraints_).then( - function success(stream) { - if (!me.mediaPermissionPending_) { - // Timeout fired while the browser permission dialog was still open; - // the outer Promise is already settled — stop the orphaned stream to release hardware. - stream.getTracks().forEach(function (t) { t.stop(); }); - return; - } - // Note: 'userMediaPermissionGranted' is both sent upward (controller → platform) and - // received downward (platform → controller for native controllers). The direction is - // determined by context: outbound is sent here; inbound is handled by this event branch. - me.sendEvent_('userMediaPermissionGranted', { - constraints: me.mediaPermissionConstraints_, - }); - me.resolveMediaPermission_(stream); - }, - function failure(error) { - if (!me.mediaPermissionPending_) { - return; - } - // Native controller: platform already granted permission but stream open failed. - if (type === 'userMediaPermissionGranted') { - me.rejectMediaPermission_(error); - } else { - me.cachedMediaError_ = error; - me.sendEvent_('userMediaPermissionDenied'); - } - } - ); + if (!me.mediaPermissionUsesExchange_) { + me.startBaseMediaPermissionRequest_(type); + return; + } + // The hand-off is accepted once per request, so a duplicate hand-off cannot open a second stream. + if (me.mediaPermissionHandOffAccepted_) { + return; + } + me.mediaPermissionHandOffAccepted_ = true; + if (me.audioInputExchangeCompleted_) { + me.startLaterMediaPermissionRequest_(me.mediaPermissionRequestId_, type); + } else { + me.startFirstMediaPermissionRequest_(me.mediaPermissionRequestId_, type); + } + } else if (type === 'preferredAudioInputDevice') { + // Accepted only while awaited. A reply meant for an earlier request is ignored: the platform's messages arrive + // in order, so it arrives before this request's hand-off, while no reply is awaited. + if (me.preferredAudioInputDeviceHandler_) { + me.preferredAudioInputDeviceHandler_(data.data && data.data.deviceId); + } } } }; diff --git a/tests/spec/methods/spec-usermedia-permissions.js b/tests/spec/methods/spec-usermedia-permissions.js index 533ab5f..ff7beba 100644 --- a/tests/spec/methods/spec-usermedia-permissions.js +++ b/tests/spec/methods/spec-usermedia-permissions.js @@ -9,8 +9,17 @@ function testUserMediaPermissions() { airconsole.device_id = DEVICE_ID; // 2 = controller airconsole.devices[0] = {}; airconsole.devices[DEVICE_ID] = { uid: 1237, nicktype: 'Sergio', location: LOCATION, custom: {} }; + spyOn(navigator.mediaDevices, 'enumerateDevices').and.returnValue(Promise.resolve(FAKE_DEVICES)); } + const PERMISSION_TIMEOUT = 60001; + + const FAKE_DEVICES = [ + { kind: 'audioinput', deviceId: 'mic-builtin', label: 'iPhone Microphone', groupId: 'g1' }, + { kind: 'audioinput', deviceId: 'mic-car', label: 'CarPlay', groupId: 'g2' }, + { kind: 'audiooutput', deviceId: 'speaker', label: 'Speaker', groupId: 'g1' }, + ]; + function teardown() { if (airconsole) { window.removeEventListener('message', airconsole.messageEventListener_); @@ -25,16 +34,19 @@ function testUserMediaPermissions() { }); } - function makeFakeStream() { + function makeFakeStream(deviceId) { + const track = { + readyState: 'live', + stop: jasmine.createSpy('stop').and.callFake(function () { track.readyState = 'ended'; }), + getSettings: function () { return { deviceId: deviceId || 'mic-car' }; }, + }; return { + track: track, getAudioTracks: function() { - return [{}]; + return [track]; }, getTracks: function() { - return [{ - stop: () => { - } - }]; + return [track]; }, }; } @@ -763,4 +775,461 @@ function testUserMediaPermissions() { dispatchCustomMessageEvent({ action: 'event', type: 'userMediaPermissionGranted' }); }); }); + + // --- Group 15: Preferred audio input device exchange --- + // First request of a game: stream #1, the request with the device list, then keep / reopen / stream #2. + // Later request: no open on the hand-off, a list-less request, then a single open. + + describe('preferred audio input device exchange', function () { + var sent; + var reply; // undefined: no automatic reply; null or a device id: answered at once + + function eventsOfType(type) { + return sent.filter(function (e) { return e.type === type; }); + } + + async function settle() { + for (let i = 0; i < 20; i++) { + await Promise.resolve(); + } + } + + function handOff(type) { + dispatchCustomMessageEvent({ action: 'event', type: type || 'promptUserMediaPermission' }); + } + + function replyPreferred(deviceId) { + dispatchCustomMessageEvent({ action: 'event', type: 'preferredAudioInputDevice', data: { deviceId: deviceId } }); + } + + function updateScreen(deviceData) { + dispatchCustomMessageEvent({ action: 'update', device_id: AirConsole.SCREEN, device_data: deviceData }); + } + + function spyGetUserMediaSequence(results) { + var i = 0; + spyOn(navigator.mediaDevices, 'getUserMedia').and.callFake(function () { + const result = results[Math.min(i, results.length - 1)]; + i += 1; + return result instanceof Error || result instanceof DOMException ? Promise.reject(result) : result; + }); + } + + beforeEach(function () { + initAirConsoleAsController(); + // The platform announced the exchange in ready (see "ready flag" below). + airconsole.audioInputExchangeEnabled_ = true; + sent = []; + reply = null; + spyOn(airconsole, 'sendEvent_').and.callFake(function (type, data) { + sent.push({ type: type, data: data }); + if (type === 'requestPreferredAudioInputDevice' && reply !== undefined) { + replyPreferred(reply); + } + }); + }); + + afterEach(teardown); + + describe('ready flag', function () { + function dispatchReady(extra) { + dispatchCustomMessageEvent(Object.assign({ + action: 'ready', + code: 1237, + device_id: DEVICE_ID, + devices: [{}, undefined, airconsole.devices[DEVICE_ID]], + }, extra)); + } + + it('Should leave the exchange off when ready does not announce it', function () { + dispatchReady({}); + expect(airconsole.audioInputExchangeEnabled_).toBe(false); + }); + + it('Should turn the exchange on when ready announces it', function () { + airconsole.audioInputExchangeEnabled_ = false; + dispatchReady({ audioInputExchangeEnabled: true }); + expect(airconsole.audioInputExchangeEnabled_).toBe(true); + }); + + it('Should open on the hand-off and send none of the new events without the flag', async function () { + airconsole.audioInputExchangeEnabled_ = false; + const stream = makeFakeStream('mic-car'); + spyGetUserMediaSequence([Promise.resolve(stream)]); + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + expect(await result).toBe(stream); + await settle(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); + expect(navigator.mediaDevices.enumerateDevices).not.toHaveBeenCalled(); + expect(eventsOfType('requestPreferredAudioInputDevice').length).toBe(0); + expect(eventsOfType('audioInputDevicesReported').length).toBe(0); + expect(eventsOfType('userMediaRequestFailed').length).toBe(0); + }); + + it('Should keep the 45 s timeout running through a screen pause without the flag', async function () { + jasmine.clock().install(); + try { + airconsole.audioInputExchangeEnabled_ = false; + const result = airconsole.getUserMedia({ audio: true }).catch(function (error) { return error; }); + updateScreen({ paused: { state: true, reason: 'audio-focus-loss' } }); + jasmine.clock().tick(45001); + expect((await result).message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + expect(eventsOfType('userMediaRequestFailed').length).toBe(0); + } finally { + jasmine.clock().uninstall(); + } + }); + + it('Should reject on destroy() without userMediaRequestFailed when the flag is off', async function () { + airconsole.audioInputExchangeEnabled_ = false; + const result = airconsole.getUserMedia({ audio: true }).catch(function (error) { return error; }); + airconsole.destroy(); + expect((await result).message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + expect(eventsOfType('userMediaRequestFailed').length).toBe(0); + }); + }); + + describe('first request', function () { + it('Should ask with the device list and keep stream #1 on a null reply', async function () { + const stream1 = makeFakeStream('mic-car'); + spyGetUserMediaSequence([Promise.resolve(stream1)]); + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + expect(await result).toBe(stream1); + await settle(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); + expect(eventsOfType('requestPreferredAudioInputDevice')[0].data).toEqual({ + devices: [ + { deviceId: 'mic-builtin', label: 'iPhone Microphone' }, + { deviceId: 'mic-car', label: 'CarPlay' }, + ], + activeDeviceId: 'mic-car', + }); + expect(eventsOfType('audioInputDevicesReported')[0].data).toEqual({ + devices: [ + { deviceId: 'mic-builtin', label: 'iPhone Microphone' }, + { deviceId: 'mic-car', label: 'CarPlay' }, + ], + activeDeviceId: 'mic-car', + }); + expect(stream1.track.stop).not.toHaveBeenCalled(); + }); + + it('Should keep stream #1 when the reply names the device it is on', async function () { + reply = 'mic-car'; + const stream1 = makeFakeStream('mic-car'); + spyGetUserMediaSequence([Promise.resolve(stream1)]); + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + expect(await result).toBe(stream1); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); + }); + + it('Should stop stream #1, then open stream #2 with the non-exact preferred deviceId', async function () { + reply = 'mic-builtin'; + const stream1 = makeFakeStream('mic-car'); + const stream2 = makeFakeStream('mic-builtin'); + spyGetUserMediaSequence([Promise.resolve(stream1), Promise.resolve(stream2)]); + const result = airconsole.getUserMedia({ audio: { echoCancellation: true } }); + handOff(); + expect(await result).toBe(stream2); + await settle(); + expect(stream1.track.stop).toHaveBeenCalled(); + expect(navigator.mediaDevices.getUserMedia.calls.argsFor(1)[0]).toEqual({ + audio: { echoCancellation: true, deviceId: 'mic-builtin' }, + }); + expect(eventsOfType('audioInputDevicesReported')[0].data.activeDeviceId).toBe('mic-builtin'); + }); + + it('Should open a fresh stream when stream #1 ended before the reply', async function () { + reply = undefined; + const stream1 = makeFakeStream('mic-car'); + const fresh = makeFakeStream('mic-builtin'); + spyGetUserMediaSequence([Promise.resolve(stream1), Promise.resolve(fresh)]); + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + await settle(); + stream1.track.readyState = 'ended'; + replyPreferred(null); + expect(await result).toBe(fresh); + expect(navigator.mediaDevices.getUserMedia.calls.argsFor(1)[0]).toEqual({ audio: true }); + }); + + it('Should send userMediaRequestFailed(open-error) when stream #2 fails, and no audioInputDevicesReported', async function () { + reply = 'mic-builtin'; + const openError = new DOMException('Busy', 'NotReadableError'); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), openError]); + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + await expectAsync(result).toBeRejectedWith(openError); + await settle(); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ + reason: 'open-error', + error: 'NotReadableError', + }); + expect(eventsOfType('audioInputDevicesReported').length).toBe(0); + }); + + it('Should take the denial path when stream #2 fails with NotAllowedError on web', async function () { + reply = 'mic-builtin'; + const notAllowed = makeNotAllowedError(); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), notAllowed]); + const result = airconsole.getUserMedia({ audio: true }); + handOff('promptUserMediaPermission'); + await settle(); + expect(eventsOfType('userMediaPermissionDenied').length).toBe(1); + expect(eventsOfType('userMediaRequestFailed').length).toBe(0); + dispatchCustomMessageEvent({ action: 'event', type: 'userMediaPermissionDenied' }); + await expectAsync(result).toBeRejectedWith(notAllowed); + }); + + it('Should send userMediaRequestFailed(open-error) when stream #2 fails with NotAllowedError in the app', async function () { + reply = 'mic-builtin'; + const notAllowed = makeNotAllowedError(); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), notAllowed]); + const result = airconsole.getUserMedia({ audio: true }); + handOff('userMediaPermissionGranted'); + await expectAsync(result).toBeRejectedWith(notAllowed); + await settle(); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ reason: 'open-error', error: 'NotAllowedError' }); + expect(eventsOfType('userMediaPermissionDenied').length).toBe(0); + }); + + it('Should send userMediaRequestFailed(open-error) when stream #1 fails after the native grant', async function () { + const openError = new DOMException('No device', 'NotFoundError'); + spyGetUserMediaSequence([openError]); + const result = airconsole.getUserMedia({ audio: true }); + handOff('userMediaPermissionGranted'); + await expectAsync(result).toBeRejectedWith(openError); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ reason: 'open-error', error: 'NotFoundError' }); + expect(eventsOfType('userMediaPermissionDenied').length).toBe(0); + }); + + it('Should accept the permission hand-off only once per request', async function () { + reply = undefined; + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car'))]); + airconsole.getUserMedia({ audio: true }).catch(function () {}); + handOff('promptUserMediaPermission'); + await settle(); + handOff('userMediaPermissionGranted'); + await settle(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); + }); + + it('Should ignore a preferredAudioInputDevice that is not awaited', async function () { + reply = undefined; + const stream1 = makeFakeStream('mic-car'); + spyGetUserMediaSequence([Promise.resolve(stream1), Promise.resolve(makeFakeStream('stray'))]); + const result = airconsole.getUserMedia({ audio: true }); + replyPreferred('stray'); + handOff(); + await settle(); + replyPreferred(null); + replyPreferred('stray'); + expect(await result).toBe(stream1); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); + }); + }); + + describe('later request', function () { + async function completeFirstRequest(type) { + const result = airconsole.getUserMedia({ audio: true }); + handOff(type); + await result; + await settle(); + sent = []; + } + + it('Should not open on the hand-off, ask without a device list, then open once with the saved device', async function () { + const later = makeFakeStream('mic-builtin'); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), Promise.resolve(later)]); + await completeFirstRequest(); + + reply = undefined; + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + await settle(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(1); + expect(eventsOfType('requestPreferredAudioInputDevice')[0].data).toEqual({}); + + replyPreferred('mic-builtin'); + expect(await result).toBe(later); + await settle(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(2); + expect(navigator.mediaDevices.getUserMedia.calls.argsFor(1)[0]).toEqual({ audio: { deviceId: 'mic-builtin' } }); + expect(eventsOfType('audioInputDevicesReported')[0].data.activeDeviceId).toBe('mic-builtin'); + }); + + it('Should take the denial path on NotAllowedError', async function () { + const notAllowed = makeNotAllowedError(); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), notAllowed]); + await completeFirstRequest(); + + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + await settle(); + expect(eventsOfType('userMediaPermissionDenied').length).toBe(1); + expect(eventsOfType('userMediaRequestFailed').length).toBe(0); + dispatchCustomMessageEvent({ action: 'event', type: 'userMediaPermissionDenied' }); + await expectAsync(result).toBeRejectedWith(notAllowed); + }); + + it('Should send userMediaRequestFailed(open-error) on NotAllowedError in the app, with no denial', async function () { + const notAllowed = makeNotAllowedError(); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), notAllowed]); + await completeFirstRequest('userMediaPermissionGranted'); + + const result = airconsole.getUserMedia({ audio: true }); + handOff('userMediaPermissionGranted'); + await expectAsync(result).toBeRejectedWith(notAllowed); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ reason: 'open-error', error: 'NotAllowedError' }); + expect(eventsOfType('userMediaPermissionDenied').length).toBe(0); + }); + + it('Should send userMediaRequestFailed(open-error) on any other error', async function () { + const busy = new DOMException('Busy', 'NotReadableError'); + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), busy]); + await completeFirstRequest(); + + const result = airconsole.getUserMedia({ audio: true }); + handOff(); + await expectAsync(result).toBeRejectedWith(busy); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ reason: 'open-error', error: 'NotReadableError' }); + }); + }); + + describe('permission timeout', function () { + beforeEach(function () { + jasmine.clock().install(); + jasmine.clock().mockDate(new Date(2026, 0, 1)); + }); + + afterEach(function () { + jasmine.clock().uninstall(); + }); + + it('Should stop stream #1 and send userMediaRequestFailed(timeout) before rejecting', async function () { + reply = undefined; + const stream1 = makeFakeStream('mic-car'); + spyGetUserMediaSequence([Promise.resolve(stream1)]); + const order = []; + airconsole.sendEvent_.and.callFake(function (type, data) { + sent.push({ type: type, data: data }); + order.push(type); + }); + const result = airconsole.getUserMedia({ audio: true }).catch(function (error) { + order.push('rejected'); + return error; + }); + handOff(); + await settle(); + jasmine.clock().tick(PERMISSION_TIMEOUT); + const error = await result; + expect(error.message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + expect(stream1.track.stop).toHaveBeenCalled(); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ reason: 'timeout', error: 'Timeout' }); + expect(order.indexOf('userMediaRequestFailed')).toBeLessThan(order.indexOf('rejected')); + }); + + it('Should end a request on destroy() like a timeout: stop stream #1, send, then reject, once', async function () { + reply = undefined; + const stream1 = makeFakeStream('mic-car'); + spyGetUserMediaSequence([Promise.resolve(stream1)]); + const order = []; + airconsole.sendEvent_.and.callFake(function (type, data) { + sent.push({ type: type, data: data }); + order.push(type); + }); + const result = airconsole.getUserMedia({ audio: true }).catch(function (error) { + order.push('rejected'); + return error; + }); + handOff(); + await settle(); + expect(eventsOfType('requestPreferredAudioInputDevice').length).toBe(1); + airconsole.destroy(); + const error = await result; + expect(error.message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + expect(stream1.track.stop).toHaveBeenCalled(); + expect(eventsOfType('userMediaRequestFailed')[0].data).toEqual({ reason: 'timeout', error: 'Timeout' }); + expect(order.indexOf('userMediaRequestFailed')).toBeLessThan(order.indexOf('rejected')); + jasmine.clock().tick(PERMISSION_TIMEOUT); + expect(eventsOfType('userMediaRequestFailed').length).toBe(1); + }); + + it('Should use the first-request flow again after a timeout before the reply, and ignore the late reply', async function () { + reply = undefined; + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car'))]); + const first = airconsole.getUserMedia({ audio: true }).catch(function () {}); + handOff(); + await settle(); + jasmine.clock().tick(PERMISSION_TIMEOUT); + await first; + replyPreferred('mic-builtin'); // crosses the timeout: ignored + + airconsole.getUserMedia({ audio: true }).catch(function () {}); + handOff(); + await settle(); + // First-request flow: opens on the hand-off and asks with a device list. + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(2); + expect(eventsOfType('requestPreferredAudioInputDevice')[1].data.devices.length).toBe(2); + }); + + it('Should use the later-request flow after a timeout that came after the reply', async function () { + reply = 'mic-builtin'; + spyGetUserMediaSequence([Promise.resolve(makeFakeStream('mic-car')), new Promise(function () {})]); + const first = airconsole.getUserMedia({ audio: true }).catch(function () {}); + handOff(); + await settle(); + jasmine.clock().tick(PERMISSION_TIMEOUT); + await first; + + reply = undefined; + airconsole.getUserMedia({ audio: true }).catch(function () {}); + handOff(); + await settle(); + expect(navigator.mediaDevices.getUserMedia).toHaveBeenCalledTimes(2); + expect(eventsOfType('requestPreferredAudioInputDevice')[1].data).toEqual({}); + }); + + it('Should stop the clock while the screen is paused and keep the remaining time', async function () { + const result = airconsole.getUserMedia({ audio: true }).catch(function (error) { return error; }); + jasmine.clock().tick(30000); + updateScreen({ paused: { state: true, reason: 'visibility' } }); + jasmine.clock().tick(120000); + await settle(); + expect(airconsole.mediaPermissionPending_).toBe(true); + updateScreen({ paused: { state: false, reason: 'resume' } }); + jasmine.clock().tick(29999); + expect(airconsole.mediaPermissionPending_).toBe(true); + jasmine.clock().tick(2); + expect((await result).message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + }); + + it('Should wait with the clock stopped when the request starts while the screen is paused', async function () { + airconsole.devices[AirConsole.SCREEN] = { paused: { state: true, reason: 'audio-focus-loss' } }; + const result = airconsole.getUserMedia({ audio: true }).catch(function (error) { return error; }); + jasmine.clock().tick(120000); + expect(airconsole.mediaPermissionPending_).toBe(true); + updateScreen({ paused: { state: false, reason: 'resume' } }); + jasmine.clock().tick(PERMISSION_TIMEOUT); + expect((await result).message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + }); + + it('Should run normally on a screen that has never paused', async function () { + airconsole.devices[AirConsole.SCREEN] = {}; + const first = airconsole.getUserMedia({ audio: true }).catch(function (error) { return error; }); + updateScreen({ location: LOCATION }); + jasmine.clock().tick(PERMISSION_TIMEOUT); + expect((await first).message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + + airconsole.devices[AirConsole.SCREEN] = undefined; + const second = airconsole.getUserMedia({ audio: true }).catch(function (error) { return error; }); + expect(airconsole.mediaPermissionPending_).toBe(true); + jasmine.clock().tick(PERMISSION_TIMEOUT); + expect((await second).message).toBe(AirConsole.USER_MEDIA_ERROR_TYPE.timeout); + }); + }); + }); }