From 23394cb345c1fc36b456acb6161b5bfcb5774bce Mon Sep 17 00:00:00 2001 From: Maryam Kia Date: Tue, 1 Aug 2023 11:28:48 +0200 Subject: [PATCH] [IMP] mail: Allow screen and video sharing at the same time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before: - sharing the screen stops sharing video and vice versa after: - If user starts to share screen at the same time with video sharing, a pip of video sharing appears on the call when user focuses on the sharing screen and vice versa. This pip is draggable and depends on the nearest corner to drag situation, will be located on the corner. Task-3374606 closes odoo/odoo#130354 Signed-off-by: Alexandre Kühn (aku) --- .../static/src/core/common/thread_model.js | 4 +- .../static/src/discuss/call/common/call.js | 98 ++++-- .../static/src/discuss/call/common/call.scss | 3 +- .../static/src/discuss/call/common/call.xml | 11 +- .../call/common/call_participant_card.js | 106 ++++++- .../call/common/call_participant_card.scss | 24 ++ .../call/common/call_participant_card.xml | 10 +- .../call/common/call_participant_video.js | 12 +- .../src/discuss/call/common/rtc_service.js | 288 +++++++++++++----- .../discuss/call/common/rtc_session_model.js | 4 +- .../static/tests/discuss/call/call_tests.js | 3 +- 11 files changed, 455 insertions(+), 108 deletions(-) diff --git a/addons/mail/static/src/core/common/thread_model.js b/addons/mail/static/src/core/common/thread_model.js index ebd20c994db..57f400fa007 100644 --- a/addons/mail/static/src/core/common/thread_model.js +++ b/addons/mail/static/src/core/common/thread_model.js @@ -107,7 +107,7 @@ export class Thread extends Record { channel; /** @type {import("@mail/core/common/channel_member_model").ChannelMember[]} */ channelMembers = []; - /** @type {RtcSession{}} */ + /** @type {Object} */ rtcSessions = {}; invitingRtcSessionId; /** @type {Set} */ @@ -481,7 +481,7 @@ export class Thread extends Record { get videoCount() { return Object.values(this._store.RtcSession.records).filter( - (session) => session.videoStream + (session) => session.videoStreams.size ).length; } diff --git a/addons/mail/static/src/discuss/call/common/call.js b/addons/mail/static/src/discuss/call/common/call.js index 72f2d4f6f33..e44f048993a 100644 --- a/addons/mail/static/src/discuss/call/common/call.js +++ b/addons/mail/static/src/discuss/call/common/call.js @@ -18,6 +18,20 @@ import { browser } from "@web/core/browser/browser"; import { _t } from "@web/core/l10n/translation"; import { useService } from "@web/core/utils/hooks"; +/** + * @typedef CardData + * @property {string} key + * @property {import("@mail/discuss/call/common/rtc_session_model").RtcSession} session + * @property {MediaStream} videoStream + * @property {import("@mail/core/common/channel_member_model").ChannelMember} [member] + */ + +/** + * @typedef {Object} Props + * @property {import("@mail/core/common/thread_model").Thread} thread + * @property {boolean} [compact] + * @extends {Component} + */ export class Call extends Component { static components = { CallActionList, CallParticipantCard }; static props = ["thread", "compact?"]; @@ -36,6 +50,8 @@ export class Call extends Component { tileHeight: 0, columnCount: 0, overlay: false, + /** @type {CardData|undefined} */ + insetCard: undefined, }); this.store = useState(useService("mail.store")); this.userSettings = useState(useService("mail.user_settings")); @@ -66,22 +82,29 @@ export class Call extends Component { return false; } + /** @returns {CardData[]} */ get visibleCards() { const raisingHandCards = []; const sessionCards = []; const invitationCards = []; const filterVideos = this.props.thread.showOnlyVideo && this.props.thread.videoCount > 0; for (const session of Object.values(this.props.thread.rtcSessions)) { - if (!filterVideos || session.videoStream) { - const data = { - key: "session_" + session.id, + const target = session.raisingHand ? raisingHandCards : sessionCards; + const cameraStream = session.videoStreams.get("camera"); + if (!filterVideos || cameraStream) { + target.push({ + key: "session_main_" + session.id, session, - }; - if (session.raisingHand) { - raisingHandCards.push(data); - } else { - sessionCards.push(data); - } + videoStream: cameraStream, + }); + } + const screenStream = session.videoStreams.get("screen"); + if (screenStream) { + target.push({ + key: "session_secondary_" + session.id, + session, + videoStream: screenStream, + }); } } if (!filterVideos) { @@ -108,16 +131,57 @@ export class Call extends Component { return raisingHandCards.concat(sessionCards, invitationCards); } + /** @returns {CardData[]} */ get visibleMainCards() { - if (this.props.thread.activeRtcSession) { - return [ - { - key: "session_" + this.props.thread.activeRtcSession.id, - session: this.props.thread.activeRtcSession, - }, - ]; + const activeSession = this.props.thread.activeRtcSession; + if (!activeSession) { + this.state.insetCard = undefined; + return this.visibleCards; } - return this.visibleCards; + switch (activeSession.videoStreams.size) { + case 1: + if (activeSession.videoStreams.get("screen") === activeSession.mainVideoStream) { + this.setInset(activeSession); + } + if ( + activeSession.videoStreams.get("camera") && + activeSession.videoStreams.get("camera") === activeSession.mainVideoStream + ) { + this.state.insetCard = undefined; + } + if (!activeSession.mainVideoStream && activeSession.videoStreams.get("screen")) { + this.setInset(activeSession, "screen"); + } + break; + case 2: { + const videoType = + activeSession.videoStreams.get("screen") === activeSession.mainVideoStream + ? "camera" + : "screen"; + this.setInset(activeSession, videoType); + break; + } + } + + return [ + { + key: "session_" + activeSession.id, + session: activeSession, + videoStream: activeSession.mainVideoStream, + }, + ]; + } + + /** + * @param {RtcSession} session + * @param {String} videoType + */ + setInset(session, videoType) { + this.state.insetCard = { + key: "session_" + session.id, + session, + videoStream: session.videoStreams.get(videoType), + }; } get hasCallNotifications() { diff --git a/addons/mail/static/src/discuss/call/common/call.scss b/addons/mail/static/src/discuss/call/common/call.scss index 6b52d85bc06..ba0b65445f3 100644 --- a/addons/mail/static/src/discuss/call/common/call.scss +++ b/addons/mail/static/src/discuss/call/common/call.scss @@ -32,7 +32,7 @@ aspect-ratio: 16/9; } -.o-discuss-Call-mainCard { +.o-discuss-Call-mainCardStyle { width: var(--width); height: var(--height); min-width: var(--width); @@ -55,6 +55,7 @@ &::-webkit-scrollbar { background: map-get($grays, '900'); } + &::-webkit-scrollbar-thumb { background: map-get($grays, '700'); } diff --git a/addons/mail/static/src/discuss/call/common/call.xml b/addons/mail/static/src/discuss/call/common/call.xml index 22931e3fd11..086bc082d26 100644 --- a/addons/mail/static/src/discuss/call/common/call.xml +++ b/addons/mail/static/src/discuss/call/common/call.xml @@ -5,7 +5,7 @@
@@ -40,6 +40,13 @@ thread="props.thread" />
+
diff --git a/addons/mail/static/src/discuss/call/common/call_participant_card.js b/addons/mail/static/src/discuss/call/common/call_participant_card.js index c4020d89dfc..07dff85d8ea 100644 --- a/addons/mail/static/src/discuss/call/common/call_participant_card.js +++ b/addons/mail/static/src/discuss/call/common/call_participant_card.js @@ -4,8 +4,16 @@ import { CallContextMenu } from "@mail/discuss/call/common/call_context_menu"; import { CallParticipantVideo } from "@mail/discuss/call/common/call_participant_video"; import { useHover } from "@mail/utils/common/hooks"; import { isEventHandled, markEventHandled } from "@web/core/utils/misc"; +import { browser } from "@web/core/browser/browser"; -import { Component, onMounted, onWillUnmount, useRef, useState } from "@odoo/owl"; +import { + Component, + onMounted, + onWillUnmount, + useRef, + useState, + useExternalListener, +} from "@odoo/owl"; import { usePopover } from "@web/core/popover/popover_hook"; import { useService } from "@web/core/utils/hooks"; @@ -13,18 +21,21 @@ import { useService } from "@web/core/utils/hooks"; const HIDDEN_CONNECTION_STATES = new Set(["connected", "completed"]); export class CallParticipantCard extends Component { - static props = ["className", "cardData", "thread", "minimized?"]; + static props = ["className", "cardData", "thread", "minimized?", "inset?"]; static components = { CallParticipantVideo }; static template = "discuss.CallParticipantCard"; setup() { this.contextMenuAnchorRef = useRef("contextMenuAnchor"); + this.root = useRef("root"); this.popover = usePopover(CallContextMenu); this.rpc = useService("rpc"); this.rtc = useState(useService("discuss.rtc")); this.store = useState(useService("mail.store")); + this.ui = useState(useService("ui")); this.rootHover = useHover("root"); this.threadService = useService("mail.thread"); + this.state = useState({ drag: false, dragPos: undefined }); onMounted(() => { if (!this.rtcSession) { return; @@ -41,6 +52,7 @@ export class CallParticipantCard extends Component { viewCountIncrement: -1, }); }); + useExternalListener(browser, "fullscreenchange", this.onFullScreenChange); } get isContextMenuAvailable() { @@ -82,23 +94,46 @@ export class CallParticipantCard extends Component { } get hasVideo() { - return Boolean(this.rtcSession?.videoStream); + return Boolean(this.props.cardData.videoStream); } get isTalking() { return Boolean(this.rtcSession && this.rtcSession.isTalking && !this.rtcSession.isMute); } + get hasRaisingHand() { + const screenStream = this.rtcSession.videoStreams.get("screen"); + return Boolean( + this.rtcSession.raisingHand && + (!screenStream || screenStream !== this.props.cardData.videoStream) + ); + } + async onClick(ev) { if (isEventHandled(ev, "CallParticipantCard.clickVolumeAnchor")) { return; } + if (this.state.drag) { + this.state.drag = false; + return; + } if (this.rtcSession) { const channel = this.rtcSession.channel; - if (this.rtcSession.eq(channel.activeRtcSession)) { + if (this.rtcSession.eq(channel.activeRtcSession) && !this.props.inset) { channel.activeRtcSession = undefined; + this.rtcSession.mainVideoStream = undefined; } else { + const activeRtcSession = channel.activeRtcSession; + const mainVideoStream = this.rtcSession.mainVideoStream; channel.activeRtcSession = this.rtcSession; + this.rtcSession.mainVideoStream = this.props.cardData.videoStream; + if (this.props.inset && activeRtcSession) { + const videoType = + activeRtcSession.videoStreams.get("camera") === mainVideoStream + ? "camera" + : "screen"; + this.props.inset(activeRtcSession, videoType); + } } return; } @@ -132,4 +167,67 @@ export class CallParticipantCard extends Component { rtcSession: this.rtcSession, }); } + + onMouseDown() { + if (!this.props.inset) { + return; + } + const onMousemove = (ev) => this.drag(ev); + const onMouseup = () => { + const insetEl = this.root.el; + if (parseInt(insetEl.style.left) < insetEl.parentNode.offsetWidth / 2) { + insetEl.style.left = "1vh"; + insetEl.style.right = ""; + } else { + insetEl.style.left = ""; + insetEl.style.right = "1vh"; + } + if (parseInt(insetEl.style.top) < insetEl.parentNode.offsetHeight / 2) { + insetEl.style.top = "1vh"; + insetEl.style.bottom = ""; + } else { + insetEl.style.bottom = "1vh"; + insetEl.style.top = ""; + } + document.removeEventListener("mouseup", onMouseup); + document.removeEventListener("mousemove", onMousemove); + }; + document.addEventListener("mouseup", onMouseup); + document.addEventListener("mousemove", onMousemove); + } + + drag(ev) { + this.state.drag = true; + const insetEl = this.root.el; + const parent = insetEl.parentNode; + const clientX = ev.clientX ?? ev.touches[0].clientX; + const clientY = ev.clientY ?? ev.touches[0].clientY; + if (!this.state.dragPos) { + this.state.dragPos = { posX: clientX, posY: clientY }; + } + const dX = this.state.dragPos.posX - clientX; + const dY = this.state.dragPos.posY - clientY; + this.state.dragPos.posX = Math.min( + Math.max(clientX, parent.offsetLeft), + parent.offsetLeft + parent.offsetWidth - insetEl.clientWidth + ); + this.state.dragPos.posY = Math.min( + Math.max(clientY, parent.offsetTop), + parent.offsetTop + parent.offsetHeight - insetEl.clientHeight + ); + insetEl.style.left = + Math.min( + Math.max(insetEl.offsetLeft - dX, 0), + parent.offsetWidth - insetEl.clientWidth + ) + "px"; + insetEl.style.top = + Math.min( + Math.max(insetEl.offsetTop - dY, 0), + parent.offsetHeight - insetEl.clientHeight + ) + "px"; + } + + onFullScreenChange() { + this.root.el.style = "left:''; top:''"; + } } diff --git a/addons/mail/static/src/discuss/call/common/call_participant_card.scss b/addons/mail/static/src/discuss/call/common/call_participant_card.scss index c0d8509a538..88108c531be 100644 --- a/addons/mail/static/src/discuss/call/common/call_participant_card.scss +++ b/addons/mail/static/src/discuss/call/common/call_participant_card.scss @@ -4,6 +4,30 @@ &.o-isTalking { box-shadow: inset 0 0 0 map-get($spacers, 1) darken($o-enterprise-action-color, 5%); + + &.o-inset { + box-shadow: inset 0 0 0 map-get($spacers, 1)/2 darken($o-enterprise-action-color, 5%); + } + } + + &.o-inset { + height: 20%; + max-height: 125px !important; + right: 1vh; + bottom: 1vh; + position: absolute !important; + cursor: move !important; + + &.o-small { + width: 30%; + left: 0; + top: 0; + } + + .o-discuss-CallParticipantCard-avatar img { + max-height: #{"min(70%, 70px)"}; + max-width: #{"min(70%, 70px)"}; + } } } diff --git a/addons/mail/static/src/discuss/call/common/call_participant_card.xml b/addons/mail/static/src/discuss/call/common/call_participant_card.xml index ffe5c673576..4e9b170548e 100644 --- a/addons/mail/static/src/discuss/call/common/call_participant_card.xml +++ b/addons/mail/static/src/discuss/call/common/call_participant_card.xml @@ -6,16 +6,20 @@ t-att-class="{ 'o-isTalking': !props.minimized and isTalking, 'o-isInvitation opacity-50': !rtcSession, + 'o-inset': props.inset, + 'o-small': props.inset and (ui.isSmall or props.minimized) }" t-att-title="name" t-att-aria-label="name" t-attf-class="{{ props.className }}" t-on-click="onClick" t-on-contextmenu="onContextMenu" + t-on-mousedown="onMouseDown" + t-on-touchmove="(ev) => this.drag(ev)" t-ref="root" > - +
Avatar - + LIVE
- + diff --git a/addons/mail/static/src/discuss/call/common/call_participant_video.js b/addons/mail/static/src/discuss/call/common/call_participant_video.js index 6b179e79258..2811034e224 100644 --- a/addons/mail/static/src/discuss/call/common/call_participant_video.js +++ b/addons/mail/static/src/discuss/call/common/call_participant_video.js @@ -3,8 +3,14 @@ import { Component, onMounted, onPatched, useExternalListener, useRef, useState } from "@odoo/owl"; import { useService } from "@web/core/utils/hooks"; +/** + * @typedef {Object} Props + * @property {import("@mail/discuss/call/common/rtc_session_model").RtcSession} session + * @property {MediaStream} [videoStream] + * @extends {Component} + */ export class CallParticipantVideo extends Component { - static props = ["session"]; + static props = ["session", "videoStream?"]; static template = "discuss.CallParticipantVideo"; setup() { @@ -21,10 +27,10 @@ export class CallParticipantVideo extends Component { if (!this.root.el) { return; } - if (!this.props.session || !this.props.session.videoStream) { + if (!this.props.session || !this.props.videoStream) { this.root.el.srcObject = undefined; } else { - this.root.el.srcObject = this.props.session.videoStream; + this.root.el.srcObject = this.props.videoStream; } this.root.el.load(); } diff --git a/addons/mail/static/src/discuss/call/common/rtc_service.js b/addons/mail/static/src/discuss/call/common/rtc_service.js index ed403f4e799..ddae9b97f23 100644 --- a/addons/mail/static/src/discuss/call/common/rtc_service.js +++ b/addons/mail/static/src/discuss/call/common/rtc_service.js @@ -12,7 +12,7 @@ import { _t } from "@web/core/l10n/translation"; import { registry } from "@web/core/registry"; import { debounce } from "@web/core/utils/timing"; -const ORDERED_TRANSCEIVER_NAMES = ["audio", "video"]; +const ORDERED_TRANSCEIVER_NAMES = ["audio", "screen", "camera"]; const PEER_NOTIFICATION_WAIT_DELAY = 50; const RECOVERY_TIMEOUT = 15_000; const RECOVERY_DELAY = 3_000; @@ -112,7 +112,8 @@ export class Rtc { isPendingNotify: false, notificationsToSend: new Map(), audioTrack: undefined, - videoTrack: undefined, + cameraTrack: undefined, + screenTrack: undefined, /** * callback to properly end the audio monitoring. * If set it indicates that we are currently monitoring the local @@ -132,6 +133,7 @@ export class Rtc { outgoingSessions: new Set(), pttReleaseTimeout: undefined, sourceCameraStream: null, + sourceScreenStream: null, }); this.blurManager = undefined; this.ringingThreads = reactive([], () => this.onRingingThreadsChange()); @@ -390,8 +392,15 @@ export class Rtc { }); return; } - await this.updateRemote(session, "audio"); - await this.updateRemote(session, "video"); + if (peerConnection.getTransceivers().length === 0) { + for (const trackKind of ORDERED_TRANSCEIVER_NAMES) { + const type = ["screen", "camera"].includes(trackKind) ? "video" : trackKind; + peerConnection.addTransceiver(type); + } + } + for (const transceiverName of ORDERED_TRANSCEIVER_NAMES) { + await this.updateRemote(session, transceiverName); + } let answer; try { @@ -487,8 +496,8 @@ export class Rtc { } break; case "trackChange": { - const { isSelfMuted, isTalking, isSendingVideo, isDeaf } = payload.state; if (payload.type === "audio") { + const { isSelfMuted, isTalking, isDeaf } = payload.state; if (!session.audioStream) { return; } @@ -497,17 +506,17 @@ export class Rtc { isTalking, isDeaf, }); + return; } - if (payload.type === "video" && isSendingVideo === false) { - /** - * Since WebRTC "unified plan", the local track is tied to the - * remote transceiver.sender and not the remote track. Therefore - * when the remote track is 'ended' the local track is not 'ended' - * but only 'muted'. This is why we do not stop the local track - * until the peer is completely removed. - */ - session.videoStream = undefined; - } + /** + * Since WebRTC "unified plan", the local track is tied to the + * remote transceiver.sender and not the remote track. Therefore + * when the remote track is 'ended' the local track is not 'ended' + * but only 'muted'. This is why we do not stop the local track + * until the peer is completely removed. + */ + this.updateActiveSession(session, payload.type); + session.videoStreams.delete(payload.type); break; } } @@ -602,7 +611,7 @@ export class Rtc { async connect(session) { this.createConnection(session); for (const transceiverName of ORDERED_TRANSCEIVER_NAMES) { - await this.updateRemote(session, transceiverName, true); + await this.updateRemote(session, transceiverName); } this.state.outgoingSessions.add(session.id); } @@ -700,9 +709,12 @@ export class Rtc { }; peerConnection.ontrack = ({ transceiver, track }) => { this.log(session, `received ${track.kind} track`); + const videoType = this.getVideoType(session.peerConnection, transceiver); this.updateStream(session, track, { mute: this.state.selfSession.isDeaf, + videoType, }); + this.updateActiveSession(session, videoType, { addVideo: true }); }; const dataChannel = peerConnection.createDataChannel("notifications", { negotiated: true, @@ -748,6 +760,15 @@ export class Rtc { return peerConnection; } + /** + * @param {RTCPeerConnection} peerConnection + * @param {RTCRtpTransceiver} transceiver + */ + getVideoType(peerConnection, transceiver) { + const transceivers = peerConnection.getTransceivers(); + return ORDERED_TRANSCEIVER_NAMES[transceivers.indexOf(transceiver)]; + } + /** * @param {import("@mail/core/common/thread_model").Thread} */ @@ -1003,7 +1024,8 @@ export class Rtc { this.state.updateAndBroadcastDebounce?.cancel(); this.state.disconnectAudioMonitor?.(); this.state.audioTrack?.stop(); - this.state.videoTrack?.stop(); + this.state.cameraTrack?.stop(); + this.state.screenTrack?.stop(); this.state.notificationsToSend.clear(); closeStream(this.state.sourceCameraStream); this.state.sourceCameraStream = null; @@ -1015,7 +1037,8 @@ export class Rtc { updateAndBroadcastDebounce: undefined, disconnectAudioMonitor: undefined, outgoingSessions: new Set(), - videoTrack: undefined, + cameraTrack: undefined, + screenTrack: undefined, audioTrack: undefined, selfSession: undefined, sendCamera: false, @@ -1121,36 +1144,62 @@ export class Rtc { } switch (type) { case "camera": { + const track = this.state.cameraTrack; const sendCamera = force ?? !this.state.sendCamera; - await this.setVideo(type, sendCamera); + this.state.sendCamera = false; + await this.setVideo(track, type, sendCamera); break; } case "screen": { + const track = this.state.screenTrack; const sendScreen = force ?? !this.state.sendScreen; - await this.setVideo(type, sendScreen); + this.state.sendScreen = false; + await this.setVideo(track, type, sendScreen); break; } } if (this.state.selfSession) { - if (!this.state.videoTrack) { - this.removeVideoFromSession(this.state.selfSession); - } else { - this.updateStream(this.state.selfSession, this.state.videoTrack); + switch (type) { + case "camera": { + this.removeVideoFromSession(this.state.selfSession, "camera"); + if (this.state.cameraTrack) { + this.updateStream(this.state.selfSession, this.state.cameraTrack); + } + break; + } + case "screen": { + if (!this.state.screenTrack) { + this.removeVideoFromSession(this.state.selfSession, "screen"); + } else { + this.updateStream(this.state.selfSession, this.state.screenTrack); + } + break; + } } } for (const session of Object.values(this.state.channel.rtcSessions)) { if (session.eq(this.state.selfSession)) { continue; } - await this.updateRemote(session, "video"); + await this.updateRemote(session, type); } if (!this.state.selfSession) { return; } - this.updateAndBroadcast({ - isScreenSharingOn: !!this.state.sendScreen, - isCameraOn: !!this.state.sendCamera, - }); + switch (type) { + case "camera": { + this.updateAndBroadcast({ + isCameraOn: !!this.state.sendCamera, + }); + break; + } + case "screen": { + this.updateAndBroadcast({ + isScreenSharingOn: !!this.state.sendScreen, + }); + break; + } + } } updateAndBroadcast(data) { @@ -1182,20 +1231,29 @@ export class Rtc { /** * @param {String} type 'camera' or 'screen' */ - async setVideo(type, activateVideo = false) { - this.state.sendScreen = false; - this.state.sendCamera = false; + async setVideo(track, type, activateVideo = false) { if (this.blurManager) { this.blurManager.close(); this.blurManager = undefined; } const stopVideo = () => { - if (this.state.videoTrack) { - this.state.videoTrack.stop(); + if (track) { + track.stop(); + } + switch (type) { + case "camera": { + this.state.cameraTrack = undefined; + closeStream(this.state.sourceCameraStream); + this.state.sourceCameraStream = null; + break; + } + case "screen": { + this.state.screenTrack = undefined; + closeStream(this.state.sourceScreenStream); + this.state.sourceScreenStream = null; + break; + } } - this.state.videoTrack = undefined; - closeStream(this.state.sourceCameraStream); - this.state.sourceCameraStream = null; }; if (!activateVideo) { if (type === "screen") { @@ -1216,9 +1274,13 @@ export class Rtc { } } if (type === "screen") { - sourceStream = await browser.navigator.mediaDevices.getDisplayMedia({ - video: VIDEO_CONFIG, - }); + if (this.state.sourceScreenStream && this.state.sendScreen) { + sourceStream = this.state.sourceScreenStream; + } else { + sourceStream = await browser.navigator.mediaDevices.getDisplayMedia({ + video: VIDEO_CONFIG, + }); + } this.soundEffectsService.play("screen-sharing"); } } catch { @@ -1246,22 +1308,30 @@ export class Rtc { this.userSettingsService.useBlur = false; } } - const track = videoStream ? videoStream.getVideoTracks()[0] : undefined; + track = videoStream ? videoStream.getVideoTracks()[0] : undefined; if (track) { track.addEventListener("ended", async () => { await this.toggleVideo(type, false); }); } - // ensures that the previous stream is stopped before overwriting it - if (this.state.sourceCameraStream && sourceStream.id !== this.state.sourceCameraStream.id) { - closeStream(this.state.sourceCameraStream); + switch (type) { + case "camera": { + Object.assign(this.state, { + sourceCameraStream: sourceStream, + cameraTrack: track, + sendCamera: Boolean(type === "camera" && track), + }); + break; + } + case "screen": { + Object.assign(this.state, { + sourceScreenStream: sourceStream, + screenTrack: track, + sendScreen: Boolean(type === "screen" && track), + }); + break; + } } - Object.assign(this.state, { - sourceCameraStream: sourceStream, - videoTrack: track, - sendCamera: Boolean(type === "camera" && track), - sendScreen: Boolean(type === "screen" && track), - }); } /** @@ -1271,29 +1341,37 @@ export class Rtc { * * negotiationneeded -> offer -> answer -> ... */ - async updateRemote(session, trackKind, initTransceiver = false) { + async updateRemote(session, trackKind) { this.log(session, `updating ${trackKind} transceiver`); - const track = trackKind === "audio" ? this.state.audioTrack : this.state.videoTrack; + let track; + switch (trackKind) { + case "audio": { + track = this.state.audioTrack; + break; + } + case "camera": { + track = this.state.cameraTrack; + break; + } + case "screen": { + track = this.state.screenTrack; + break; + } + } let transceiverDirection = track ? "sendrecv" : "recvonly"; - if (trackKind === "video") { + if (trackKind !== "audio") { transceiverDirection = this.getTransceiverDirection(session, Boolean(track)); } - let transceiver; - if (initTransceiver) { - transceiver = session.peerConnection.addTransceiver(trackKind); - } else { - transceiver = getTransceiver(session.peerConnection, trackKind); - } + const transceiver = getTransceiver(session.peerConnection, trackKind); try { await transceiver.sender.replaceTrack(track || null); transceiver.direction = transceiverDirection; } catch { this.log(session, `failed to update ${trackKind} transceiver`); } - if (!track && trackKind === "video") { + if (!track && trackKind !== "audio") { this.notify([session], "trackChange", { - type: "video", - state: { isSendingVideo: false }, + type: this.getVideoType(session.peerConnection, transceiver), }); } } @@ -1410,8 +1488,9 @@ export class Rtc { * @param {MediaStreamTrack} track * @param {Object} [parm1] * @param {boolean} [parm1.mute] + * @param {"camera"|"screen"} [parm1.videoType] */ - async updateStream(session, track, { mute } = {}) { + async updateStream(session, track, { mute, videoType } = {}) { const stream = new window.MediaStream(); stream.addTrack(track); if (track.kind === "audio") { @@ -1430,13 +1509,68 @@ export class Rtc { await session.playAudio(); } if (track.kind === "video") { - session.videoStream = stream; + videoType = videoType + ? videoType + : track.id === this.state.cameraTrack?.id + ? "camera" + : "screen"; + session.videoStreams.set(videoType, stream); + this.updateActiveSession(session, videoType, { addVideo: true }); } } - removeVideoFromSession(session) { - closeStream(session.videoStream); - session.videoStream = undefined; + /** + * @param {RtcSession} session + * @param {String} [videoType] + */ + removeVideoFromSession(session, videoType = false) { + if (videoType) { + this.updateActiveSession(session, videoType); + closeStream(session.videoStreams.get(videoType)); + session.videoStreams.delete(videoType); + } else { + for (const stream of session.videoStreams.values()) { + closeStream(stream); + } + session.videoStreams.clear(); + } + } + + /** + * @param {RtcSession} session + * @param {"screen"|"camera"} [videoType] + */ + updateActiveSession(session, videoType, { addVideo = false } = {}) { + const activeRtcSession = this.state.channel.activeRtcSession; + if (addVideo) { + if (videoType === "screen") { + this.state.channel.activeRtcSession = session; + session.mainVideoStream = session.videoStreams.get("screen"); + return; + } + if (activeRtcSession && session.videoStreams.size && !session.mainVideoStream) { + session.mainVideoStream = session.videoStreams.get("camera"); + } + return; + } + if (!activeRtcSession || activeRtcSession.notEq(session)) { + return; + } + if (activeRtcSession.mainVideoStream) { + const activeVideoType = + activeRtcSession.videoStreams.get("screen") === session.mainVideoStream + ? "screen" + : "camera"; + if (videoType === activeVideoType) { + session.mainVideoStream = undefined; + if (videoType === "screen") { + this.state.channel.activeRtcSession = undefined; + } + } + } + if (!activeRtcSession.mainVideoStream && videoType === "screen") { + session.mainVideoStream = undefined; + } } updateVideoDownload(rtcSession, { viewCountIncrement }) { @@ -1445,14 +1579,20 @@ export class Rtc { return; } const transceivers = rtcSession.peerConnection.getTransceivers(); - const transceiver = transceivers[ORDERED_TRANSCEIVER_NAMES.indexOf("video")]; - if (!transceiver) { - return; + const cameraTransceiver = transceivers[ORDERED_TRANSCEIVER_NAMES.indexOf("camera")]; + const screenTransceiver = transceivers[ORDERED_TRANSCEIVER_NAMES.indexOf("screen")]; + if (cameraTransceiver) { + cameraTransceiver.direction = this.getTransceiverDirection( + rtcSession, + Boolean(this.state.cameraTrack) + ); + } + if (screenTransceiver) { + screenTransceiver.direction = this.getTransceiverDirection( + rtcSession, + Boolean(this.state.screenTrack) + ); } - transceiver.direction = this.getTransceiverDirection( - rtcSession, - Boolean(this.state.videoTrack) - ); } getTransceiverDirection(session, allowUpload = false) { diff --git a/addons/mail/static/src/discuss/call/common/rtc_session_model.js b/addons/mail/static/src/discuss/call/common/rtc_session_model.js index 9ab98099138..9fff093ad73 100644 --- a/addons/mail/static/src/discuss/call/common/rtc_session_model.js +++ b/addons/mail/static/src/discuss/call/common/rtc_session_model.js @@ -62,8 +62,10 @@ export class RtcSession extends Record { /** @type {Date|undefined} */ raisingHand; videoComponentCount = 0; + /** @type {Map<'screen'|'camera', MediaStream>} */ + videoStreams = new Map(); /** @type {MediaStream} */ - videoStream; + mainVideoStream; // RTC stats connectionState; localCandidateType; diff --git a/addons/mail/static/tests/discuss/call/call_tests.js b/addons/mail/static/tests/discuss/call/call_tests.js index a271b385b13..71bcecf2801 100644 --- a/addons/mail/static/tests/discuss/call/call_tests.js +++ b/addons/mail/static/tests/discuss/call/call_tests.js @@ -7,7 +7,7 @@ import { mockGetMedia, start } from "@mail/../tests/helpers/test_utils"; import { browser } from "@web/core/browser/browser"; import { patchWithCleanup } from "@web/../tests/helpers/utils"; -import { click, contains } from "@web/../tests/utils"; +import { click, contains, triggerEvents } from "@web/../tests/utils"; QUnit.module("call"); @@ -160,6 +160,7 @@ QUnit.test("can share screen", async () => { await click(".o-discuss-CallActionList [title='More']"); await click("[title='Share Screen']"); await contains(".o-discuss-CallParticipantCard video"); + await triggerEvents(".o-discuss-Call-mainCards", ["mousemove"]); // show overlay await click(".o-discuss-CallActionList [title='More']"); await click("[title='Stop Sharing Screen']"); await contains(".o-discuss-CallParticipantCard video", { count: 0 });