Enable screensharing in desktop app (#2042)

* enable screensharing in desktop app

* add source picker

* update dependencies

* improve source picker

* improve source picker

* update thumbnails

* decrease thumbnail size

* revert unnecessary changes in screen sharing store

* fix types and eslint

* fix prettier

* remove unused import

desktop/electron/package.json

@@ -31,13 +31,13 @@
     },
     "devDependencies": {
         "@types/auto-launch": "^5.0.2",
-        "@typescript-eslint/eslint-plugin": "^2.26.0",
+        "@typescript-eslint/eslint-plugin": "^5.18.0",
-        "@typescript-eslint/parser": "^2.26.0",
+        "@typescript-eslint/parser": "^5.18.0",
-        "electron": "^17.0.1",
+        "electron": "^18.0.3",
         "electron-builder": "^22.14.13",
-        "eslint": "^6.8.0",
+        "eslint": "^8.12.0",
-        "prettier": "^2.5.1",
+        "prettier": "^2.6.2",
-        "tsup": "^5.11.13",
+        "tsup": "^5.12.4",
-        "typescript": "^3.8.3"
+        "typescript": "^4.6.3"
     }
 }

desktop/electron/src/app.ts

@@ -10,7 +10,7 @@ import { setLogLevel } from "./log";
 import "./serve"; // prepare custom url scheme
 import { loadShortcuts } from "./shortcuts";
 
-function init() {
+async function init() {
     const appLock = app.requestSingleInstanceLock();
 
     if (!appLock) {
@@ -21,7 +21,7 @@ function init() {
 
     app.on("second-instance", () => {
         // re-create window if closed
-        createWindow();
+        void createWindow();
 
         const mainWindow = getWindow();
 
@@ -36,15 +36,15 @@ function init() {
     });
 
     // This method will be called when Electron has finished loading
-    app.whenReady().then(async () => {
+    await app.whenReady().then(async () => {
         await settings.init();
 
         setLogLevel(settings.get("log_level") || "info");
 
-        autoUpdater.init();
+        await autoUpdater.init();
 
         // enable auto launch
-        updateAutoLaunch();
+        await updateAutoLaunch();
 
         // load ipc handler
         ipc();
@@ -72,7 +72,7 @@ function init() {
         // On macOS it's common to re-create a window in the app when the
         // dock icon is clicked and there are no other windows open.
         if (BrowserWindow.getAllWindows().length === 0) {
-            createWindow();
+            void createWindow();
         }
     });
 

desktop/electron/src/auto-updater.ts

@@ -38,11 +38,13 @@ export async function manualRequestUpdateCheck() {
     isManualRequestedUpdate = false;
 }
 
-function init() {
+async function init() {
     autoUpdater.logger = log;
 
-    autoUpdater.on("update-downloaded", ({ releaseNotes, releaseName }) => {
+    autoUpdater.on(
-        (async () => {
+        "update-downloaded",
+        ({ releaseNotes, releaseName }: { releaseNotes: string; releaseName: string }) => {
+            void (async () => {
                 const dialogOpts = {
                     type: "question",
                     buttons: ["Install and Restart", "Install Later"],
@@ -63,7 +65,8 @@ function init() {
                     app.quit();
                 }
             })();
-    });
+        }
+    );
 
     if (process.platform === "linux" && !process.env.APPIMAGE) {
         autoUpdater.autoDownload = false;
@@ -85,7 +88,7 @@ function init() {
         }
     });
 
-    checkForUpdates();
+    await checkForUpdates();
 
     // run update check every hour again
     setInterval(() => checkForUpdates, 1000 * 60 * 1);

desktop/electron/src/ipc.ts

@@ -1,4 +1,4 @@
-import { ipcMain, app } from "electron";
+import { ipcMain, app, desktopCapturer } from "electron";
 import electronIsDev from "electron-is-dev";
 import { createAndShowNotification } from "./notification";
 import { Server } from "./preload-local-app/types";
@@ -30,10 +30,18 @@ export default () => {
     ipcMain.handle("get-version", () => (electronIsDev ? "dev" : app.getVersion()));
 
     // app ipc
-    ipcMain.on("app:notify", (event, txt) => {
+    ipcMain.on("app:notify", (event, txt: string) => {
         createAndShowNotification({ body: txt });
     });
 
+    ipcMain.handle("app:getDesktopCapturerSources", async (event, options: Electron.SourcesOptions) => {
+        return (await desktopCapturer.getSources(options)).map((source) => ({
+            id: source.id,
+            name: source.name,
+            thumbnailURL: source.thumbnail.toDataURL(),
+        }));
+    });
+
     // local-app ipc
     ipcMain.handle("local-app:showLocalApp", () => {
         hideAppView();
@@ -43,7 +51,7 @@ export default () => {
         return settings.get("servers");
     });
 
-    ipcMain.handle("local-app:selectServer", (event, serverId: string) => {
+    ipcMain.handle("local-app:selectServer", async (event, serverId: string) => {
         const servers = settings.get("servers") || [];
         const selectedServer = servers.find((s) => s._id === serverId);
 
@@ -51,7 +59,7 @@ export default () => {
             return new Error("Server not found");
         }
 
-        showAppView(selectedServer.url);
+        await showAppView(selectedServer.url);
         return true;
     });
 

desktop/electron/src/log.ts

@@ -15,6 +15,7 @@ function onError(e: Error) {
 function onRejection(reason: Error) {
     if (reason instanceof Error) {
         let _reason = reason;
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
         const errPrototype = Object.getPrototypeOf(reason);
         const nameProperty = Object.getOwnPropertyDescriptor(errPrototype, "name");
 

desktop/electron/src/main.ts

@@ -2,4 +2,4 @@ import app from "./app";
 import log from "./log";
 
 log.init();
-app.init();
+void app.init();

desktop/electron/src/preload-app/preload.ts

@@ -8,6 +8,7 @@ const api: WorkAdventureDesktopApi = {
     notify: (txt) => ipcRenderer.send("app:notify", txt),
     onMuteToggle: (callback) => ipcRenderer.on("app:on-mute-toggle", callback),
     onCameraToggle: (callback) => ipcRenderer.on("app:on-camera-toggle", callback),
+    getDesktopCapturerSources: (options) => ipcRenderer.invoke("app:getDesktopCapturerSources", options),
 };
 
 contextBridge.exposeInMainWorld("WAD", api);

desktop/electron/src/preload-app/types.ts

@@ -1,3 +1,15 @@
+// copy of Electron.SourcesOptions to avoid Electron dependency in front
+export interface SourcesOptions {
+    types: string[];
+    thumbnailSize?: { height: number; width: number };
+}
+
+export interface DesktopCapturerSource {
+    id: string;
+    name: string;
+    thumbnailURL: string;
+}
+
 export type WorkAdventureDesktopApi = {
     desktop: boolean;
     isDevelopment: () => Promise<boolean>;
@@ -5,4 +17,5 @@ export type WorkAdventureDesktopApi = {
     notify: (txt: string) => void;
     onMuteToggle: (callback: () => void) => void;
     onCameraToggle: (callback: () => void) => void;
+    getDesktopCapturerSources: (options: SourcesOptions) => Promise<DesktopCapturerSource[]>;
 };

desktop/electron/src/tray.ts

@@ -36,14 +36,14 @@ export function createTray() {
         },
         {
             label: "Check for updates",
-            async click() {
+            click() {
-                await autoUpdater.manualRequestUpdateCheck();
+                void autoUpdater.manualRequestUpdateCheck();
             },
         },
         {
             label: "Open Logs",
             click() {
-                log.openLog();
+                void log.openLog();
             },
         },
         {

desktop/electron/src/window.ts

@@ -115,7 +115,7 @@ export async function createWindow() {
     }
 }
 
-export function showAppView(url?: string) {
+export async function showAppView(url?: string) {
     if (!appView) {
         throw new Error("App view not found");
     }
@@ -130,7 +130,7 @@ export function showAppView(url?: string) {
     mainWindow.addBrowserView(appView);
 
     if (url && url !== appViewUrl) {
-        appView.webContents.loadURL(url);
+        await appView.webContents.loadURL(url);
         appViewUrl = url;
     }
 

front/src/Api/desktop/index.ts

@@ -4,7 +4,7 @@ import { WorkAdventureDesktopApi } from "@wa-preload-app";
 
 declare global {
     interface Window {
-        WAD: WorkAdventureDesktopApi;
+        WAD?: WorkAdventureDesktopApi;
     }
 }
 

front/src/Components/MainLayout.svelte

@@ -38,6 +38,7 @@
     import { actionsMenuStore } from "../Stores/ActionsMenuStore";
     import ActionsMenu from "./ActionsMenu/ActionsMenu.svelte";
     import Lazy from "./Lazy.svelte";
+    import { showDesktopCapturerSourcePicker } from "../Stores/ScreenSharingStore";
 
     let mainLayout: HTMLDivElement;
 
@@ -119,6 +120,11 @@
 
         <Lazy when={$emoteMenuStore} component={() => import("./EmoteMenu/EmoteMenu.svelte")} />
 
+        <Lazy
+            when={$showDesktopCapturerSourcePicker}
+            component={() => import("./Video/DesktopCapturerSourcePicker.svelte")}
+        />
+
         {#if hasEmbedScreen}
             <EmbedScreensContainer />
         {/if}

front/src/Components/Video/DesktopCapturerSourcePicker.svelte

@@ -0,0 +1,171 @@
+<script lang="ts">
+    import { fly } from "svelte/transition";
+    import {
+        desktopCapturerSourcePromiseResolve,
+        showDesktopCapturerSourcePicker,
+    } from "../../Stores/ScreenSharingStore";
+    import { onDestroy, onMount } from "svelte";
+    import type { DesktopCapturerSource } from "@wa-preload-app";
+
+    let desktopCapturerSources: DesktopCapturerSource[] = [];
+    let interval: ReturnType<typeof setInterval>;
+
+    async function getDesktopCapturerSources() {
+        if (!window.WAD) {
+            throw new Error("This component can only be used in the desktop app");
+        }
+        desktopCapturerSources = await window.WAD.getDesktopCapturerSources({
+            thumbnailSize: {
+                height: 144,
+                width: 256,
+            },
+            types: ["screen", "window"],
+        });
+    }
+
+    onMount(async () => {
+        await getDesktopCapturerSources();
+        interval = setInterval(() => {
+            void getDesktopCapturerSources();
+        }, 1000);
+    });
+
+    onDestroy(() => {
+        clearInterval(interval);
+    });
+
+    function selectDesktopCapturerSource(source: DesktopCapturerSource) {
+        if (!desktopCapturerSourcePromiseResolve) {
+            throw new Error("desktopCapturerSourcePromiseResolve is not defined");
+        }
+        desktopCapturerSourcePromiseResolve(source);
+        close();
+    }
+
+    function cancel() {
+        if (!desktopCapturerSourcePromiseResolve) {
+            throw new Error("desktopCapturerSourcePromiseResolve is not defined");
+        }
+        desktopCapturerSourcePromiseResolve(null);
+        close();
+    }
+
+    function close() {
+        $showDesktopCapturerSourcePicker = false;
+    }
+</script>
+
+<div class="source-picker nes-container is-rounded" transition:fly={{ y: -50, duration: 500 }}>
+    <button type="button" class="nes-btn is-error close" on:click={cancel}>&times</button>
+    <h2>Select a Screen or Window to share!</h2>
+    <section class="streams">
+        {#each desktopCapturerSources as source}
+            <div
+                class="media-box nes-container is-rounded clickable"
+                on:click|preventDefault={() => selectDesktopCapturerSource(source)}
+            >
+                <img src={source.thumbnailURL} alt={source.name} />
+                <div class="container">
+                    {source.name}
+                </div>
+            </div>
+        {/each}
+    </section>
+</div>
+
+<style lang="scss">
+    .source-picker {
+        position: absolute;
+        pointer-events: auto;
+        background: #eceeee;
+        margin-left: auto;
+        margin-right: auto;
+        left: 0;
+        right: 0;
+        margin-top: 4%;
+        height: 80vh;
+        width: 80vw;
+        max-width: 1024px;
+        z-index: 900;
+        text-align: center;
+        display: flex;
+        flex-direction: column;
+        background-color: #333333;
+        color: whitesmoke;
+
+        .nes-btn.is-error.close {
+            position: absolute;
+            top: -20px;
+            right: -20px;
+        }
+
+        h2 {
+            font-family: "Press Start 2P";
+        }
+
+        section.streams {
+            display: flex;
+            flex-direction: row;
+            flex-wrap: wrap;
+            gap: 10px;
+            overflow-y: auto;
+            justify-content: center;
+            align-content: flex-start;
+            height: 100%;
+        }
+
+        .media-box {
+            position: relative;
+            padding: 0;
+            width: calc(100% / 3 - 20px);
+            max-width: 256px;
+            padding-bottom: calc(min((100% / 3 - 20px), 256px) * (144px / 256px));
+            max-height: 144px;
+            justify-content: center;
+            background-color: #000;
+            background-clip: padding-box;
+
+            &.clickable * {
+                cursor: url("../../../style/images/cursor_pointer.png"), pointer;
+            }
+
+            &:hover {
+                transform: scale(1.05);
+            }
+
+            img {
+                position: absolute;
+                top: 50%;
+                left: 50%;
+                max-width: 100%;
+                max-height: 100%;
+                transform: translate(-50%, -50%);
+            }
+
+            &.nes-container.is-rounded {
+                border-image-outset: 1;
+            }
+
+            div.container {
+                position: absolute;
+                width: 90%;
+                height: auto;
+                left: 5%;
+                top: calc(100% - 28px);
+                text-align: center;
+                padding: 2px 36px;
+
+                white-space: nowrap;
+                overflow-x: hidden;
+                text-overflow: ellipsis;
+                font-size: 14px;
+                margin: 2px;
+                background-color: white;
+                color: #333333;
+                border: solid 3px black;
+                border-radius: 8px;
+                font-style: normal;
+            }
+        }
+    }
+</style>

front/src/Stores/ScreenSharingStore.ts

@@ -2,6 +2,7 @@ import { derived, Readable, readable, writable } from "svelte/store";
 import { peerStore } from "./PeerStore";
 import type { LocalStreamStoreValue } from "./MediaStore";
 import { myCameraVisibilityStore } from "./MyCameraStoreVisibility";
+import type { DesktopCapturerSource } from "@wa-preload-app";
 
 declare const navigator: any; // eslint-disable-line @typescript-eslint/no-explicit-any
 
@@ -91,6 +92,25 @@ export const screenSharingConstraintsStore = derived(
     } as MediaStreamConstraints
 );
 
+async function getDesktopCapturerSources() {
+    showDesktopCapturerSourcePicker.set(true);
+    const source = await new Promise<DesktopCapturerSource | null>((resolve) => {
+        desktopCapturerSourcePromiseResolve = resolve;
+    });
+    if (source === null) {
+        return;
+    }
+    return navigator.mediaDevices.getUserMedia({
+        audio: false,
+        video: {
+            mandatory: {
+                chromeMediaSource: "desktop",
+                chromeMediaSourceId: source.id,
+            },
+        },
+    });
+}
+
 /**
  * A store containing the MediaStream object for ScreenSharing (or null if nothing requested, or Error if an error occurred)
  */
@@ -110,7 +130,9 @@ export const screenSharingLocalStreamStore = derived<Readable<MediaStreamConstra
         }
 
         let currentStreamPromise: Promise<MediaStream>;
-        if (navigator.getDisplayMedia) {
+        if (window.WAD?.getDesktopCapturerSources) {
+            currentStreamPromise = getDesktopCapturerSources();
+        } else if (navigator.getDisplayMedia) {
             currentStreamPromise = navigator.getDisplayMedia({ constraints });
         } else if (navigator.mediaDevices && navigator.mediaDevices.getDisplayMedia) {
             currentStreamPromise = navigator.mediaDevices.getDisplayMedia({ constraints });
@@ -200,3 +222,7 @@ export const screenSharingLocalMedia = readable<ScreenSharingLocalMedia | null>(
         unsubscribe();
     };
 });
+
+export const showDesktopCapturerSourcePicker = writable(false);
+
+export let desktopCapturerSourcePromiseResolve: ((source: DesktopCapturerSource | null) => void) | undefined;