feat(ScreenStream): add ScreenStream

src/components/Device.vue

@@ -44,6 +44,7 @@ import { shutdown } from "../frontcommand/scrcpyMaskCmd";
 import { useGlobalStore } from "../store/global";
 import { useI18n } from "vue-i18n";
 import { closeExternalControl, connectExternalControl } from "../websocket";
+import { LogicalSize, getCurrent } from "@tauri-apps/api/window";
 
 const { t } = useI18n();
 const dialog = useDialog();
@@ -79,7 +80,19 @@ onMounted(async () => {
           } else {
             store.screenSizeW = payload.width;
             store.screenSizeH = payload.height;
-            message.info(t("pages.Device.deviceRotation"));
+            message.info(t("pages.Device.rotation", [payload.rotation * 90]));
+          }
+          if (store.rotation.enable) {
+            let maskW: number;
+            let maskH: number;
+            if (payload.width >= payload.height) {
+              maskW = Math.round(store.rotation.horizontalLength);
+              maskH = Math.round(maskW * (payload.height / payload.width));
+            } else {
+              maskH = Math.round(store.rotation.verticalLength);
+              maskW = Math.round(maskH * (payload.width / payload.height));
+            }
+            getCurrent().setSize(new LogicalSize(maskW + 70, maskH + 30));
           }
           break;
         default:

src/components/Mask.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { h, onActivated, onMounted } from "vue";
+import { h, onActivated, onMounted, ref } from "vue";
 import { MessageReactive, NDialog, useDialog, useMessage } from "naive-ui";
 import { useGlobalStore } from "../store/global";
 import { onBeforeRouteLeave, useRouter } from "vue-router";
@@ -11,6 +11,7 @@ import {
   unlistenToEvent,
 } from "../hotkey";
 import { KeyMappingConfig, KeySteeringWheel } from "../keyMappingConfig";
+import ScreenStream from "./ScreenStream.vue";
 import { getVersion } from "@tauri-apps/api/app";
 import { fetch } from "@tauri-apps/plugin-http";
 import { open } from "@tauri-apps/plugin-shell";
@@ -25,7 +26,11 @@ const router = useRouter();
 const message = useMessage();
 const dialog = useDialog();
 
+const curPageActive = ref(false);
+const screenStreamClientId = genClientId();
+
 onBeforeRouteLeave(() => {
+  curPageActive.value = false;
   if (store.controledDevice) {
     unlistenToEvent();
     clearShortcuts();
@@ -34,13 +39,12 @@ onBeforeRouteLeave(() => {
 });
 
 onActivated(async () => {
+  curPageActive.value = true;
   cleanAfterimage();
-  const maskElement = document.getElementById("maskElement") as HTMLElement;
 
   if (store.controledDevice) {
     if (
       applyShortcuts(
-        maskElement,
         store.keyMappingConfigList[store.curKeyMappingIndex],
         store,
         message,
@@ -61,6 +65,13 @@ onMounted(async () => {
   store.checkAdb = checkAdb;
   setTimeout(() => {
     checkAdb();
+    // listen to window resize event
+    const maskElement = document.getElementById("maskElement") as HTMLElement;
+    const appWindow = getCurrent();
+    appWindow.onResized(() => {
+      store.maskSizeH = maskElement.clientHeight;
+      store.maskSizeW = maskElement.clientWidth;
+    });
   }, 500);
 });
 
@@ -79,6 +90,17 @@ async function checkAdb() {
   }
 }
 
+function genClientId() {
+  let result = "";
+  const characters =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  const charactersLength = characters.length;
+  for (let i = 0; i < 16; i++) {
+    result += characters.charAt(Math.floor(Math.random() * charactersLength));
+  }
+  return result;
+}
+
 async function loadLocalStore() {
   const localStore = new Store("store.bin");
   // loading keyMappingConfigList from local store
@@ -226,6 +248,10 @@ async function checkUpdate() {
   </div>
   <template v-if="store.keyMappingConfigList.length">
     <div @contextmenu.prevent class="mask" id="maskElement"></div>
+    <ScreenStream
+      :cid="screenStreamClientId"
+      v-if="curPageActive && store.controledDevice && store.screenStream.enable"
+    />
     <div
       v-if="store.maskButton.show"
       :style="'--transparency: ' + store.maskButton.transparency"
@@ -344,4 +370,3 @@ async function checkUpdate() {
   }
 }
 </style>
-h, import { getVersion } from "@tauri-apps/api/app";

src/components/ScreenStream.vue

@@ -0,0 +1,75 @@
+<script setup lang="ts">
+import { onBeforeUnmount, onMounted, ref } from "vue";
+import { useGlobalStore } from "../store/global";
+import { MessageReactive, useMessage } from "naive-ui";
+import { ScreenStream } from "../screenStream";
+
+const props = defineProps<{
+  cid: string;
+}>();
+
+const store = useGlobalStore();
+const message = useMessage();
+
+const streamImg = ref<HTMLImageElement | null>(null);
+
+let msgReactive: MessageReactive | null = null;
+
+function connectScreenStream() {
+  if (streamImg.value) {
+    const ss = new ScreenStream(streamImg.value, props.cid);
+    ss.connect(
+      store.screenStream.address,
+      () => {},
+      () => {
+        msgReactive = message.error("投屏连接失败。关闭此信息将尝试重新连接", {
+          duration: 0,
+          closable: true,
+          onClose: () => connectScreenStream(),
+        });
+      }
+    );
+  }
+}
+
+onMounted(() => {
+  connectScreenStream();
+});
+
+onBeforeUnmount(() => {
+  if (streamImg.value) streamImg.value.src = "";
+  if (msgReactive) {
+    msgReactive.destroy();
+  }
+});
+</script>
+
+<template>
+  <div class="screen-stream">
+    <img
+      :style="{
+        width: `${store.maskSizeW}px`,
+        height: `${store.maskSizeH}px`,
+      }"
+      ref="streamImg"
+    />
+  </div>
+</template>
+
+<style scoped lang="scss">
+.screen-stream {
+  position: absolute;
+  left: 70px;
+  top: 30px;
+  height: 100%;
+  width: 100%;
+  z-index: 0;
+  pointer-events: none;
+  user-select: none;
+
+  img {
+    pointer-events: none;
+    user-select: none;
+  }
+}
+</style>

src/hotkey.ts

@@ -1516,14 +1516,13 @@ export function clearShortcuts() {
 }
 
 export function applyShortcuts(
-  element: HTMLElement,
   keyMappingConfig: KeyMappingConfig,
   globalStore: ReturnType<typeof useGlobalStore>,
   messageAPI: ReturnType<typeof useMessage>,
   i18nT: ReturnType<typeof useI18n>["t"]
 ) {
   store = globalStore;
-  maskElement = element;
+  maskElement = document.getElementById("maskElement") as HTMLElement;
   message = messageAPI;
   t = i18nT;
 

src/i18n/en-US.json

@@ -36,11 +36,9 @@
       "wsClose": "Close",
       "wsConnect": "Control",
       "adbDeviceError": "Unable to get available devices",
-      "adbConnectError": "Wireless connection failed",
+      "adbConnectError": "Wireless connection failed"
-      "deviceRotation": "Device rotation"
     },
     "Mask": {
-      "inputBoxPlaceholder": "Input text and then press enter/esc",
       "keyconfigException": "The key mapping config is abnormal, please delete this config",
       "blankConfig": "Blank config",
       "checkUpdate": {
@@ -77,9 +75,9 @@
         "areaSaved": "Mask area saved",
         "incorrectArea": "Please enter the coordinates and size of the mask correctly",
         "buttonPrompts": "Button prompts",
-        "ifButtonPrompts": "Whether to display",
+        "ifButtonPrompts": "Show key prompts",
         "opacity": "Opacity",
-        "areaAdjust": "Mask adjustment",
+        "areaAdjust": "Mask area",
         "areaPlaceholder": {
           "x": "X coordinate of upper left corner"
         },
@@ -87,8 +85,7 @@
           "y": "Y coordinate of upper left corner",
           "w": "Mask width",
           "h": "Mask height"
-        },
+        }
-        "areaTip": "Tip: The mask size and device size will be used for coordinate conversion, please ensure the accuracy of the size"
       },
       "Basic": {
         "delLocalStore": {

src/i18n/zh-CN.json

@@ -37,7 +37,7 @@
       "wsConnect": "控制",
       "adbDeviceError": "无法获取可用设备",
       "adbConnectError": "无线连接失败",
-      "deviceRotation": "设备旋转"
+      "rotation": "设备旋转 {0}°"
     },
     "Mask": {
       "keyconfigException": "按键方案异常，请删除此方案",
@@ -56,7 +56,6 @@
         "content": "请前往设备页面，控制任意设备",
         "positiveText": "去控制"
       },
-      "inputBoxPlaceholder": "输入文本后按Enter/Esc",
       "sightMode": "鼠标已锁定, 按 {0} 键解锁",
       "checkAdb": "adb不可用，软件无法正常运行，请确保系统已安装adb，并正确添加到了Path环境变量中: {0}",
       "keyInputMode": "已进入按键输入模式，关闭本消息可退出"
@@ -71,9 +70,9 @@
         "incorrectArea": "请正确输入蒙版的坐标和尺寸",
         "areaSaved": "蒙版区域已保存",
         "buttonPrompts": "按键提示",
-        "ifButtonPrompts": "是否显示",
+        "ifButtonPrompts": "显示按键提示",
         "opacity": "不透明度",
-        "areaAdjust": "蒙版调整",
+        "areaAdjust": "蒙版区域",
         "areaPlaceholder": {
           "x": "左上角X坐标"
         },
@@ -82,7 +81,6 @@
           "w": "蒙版宽度",
           "h": "蒙版高度"
         },
-        "areaTip": "提示：蒙版尺寸与设备尺寸将用于坐标转换，请保证尺寸的准确性",
         "areaFormMissing": {
           "x": "请输入蒙版左上角X坐标",
           "y": "请输入蒙版左上角Y坐标",

src/screenStream.ts

@@ -0,0 +1,40 @@
+export class ScreenStream {
+  img: HTMLImageElement;
+  clientId: string;
+  connectTimeoutId: number | undefined;
+
+  public constructor(imgElement: HTMLImageElement, clientId: string) {
+    this.img = imgElement;
+    this.clientId = clientId;
+  }
+
+  public connect(address: string, onConnect: () => void, onError: () => void) {
+    if (address.endsWith("/")) address = address.slice(0, -1);
+    const that = this;
+    const img = that.img;
+    const url = `${address}/stream.mjpeg?clientId=${this.clientId}`;
+
+    img.src = "";
+    clearTimeout(that.connectTimeoutId);
+    new Promise<void>((resolve, reject) => {
+      img.onload = function () {
+        img.onload = null;
+        img.onerror = null;
+        resolve();
+      };
+      img.onerror = function (e) {
+        img.onerror = null;
+        img.onload = null;
+        reject(e);
+      };
+      img.src = url;
+    })
+      .then(() => {
+        onConnect();
+      })
+      .catch(() => {
+        img.src = "";
+        onError();
+      });
+  }
+}

src/store/global.ts

@@ -75,6 +75,20 @@ export const useGlobalStore = defineStore("global", () => {
 
   const keyInputFlag = ref(false);
 
+  const maskSizeW: Ref<number> = ref(0);
+  const maskSizeH: Ref<number> = ref(0);
+
+  const screenStream = ref({
+    enable: false,
+    address: "",
+  });
+
+  const rotation = ref({
+    enable: true,
+    verticalLength: 600,
+    horizontalLength: 800,
+  });
+
   // persistent storage
   const keyMappingConfigList: Ref<KeyMappingConfig[]> = ref([]);
   const curKeyMappingIndex = ref(0);
@@ -92,6 +106,10 @@ export const useGlobalStore = defineStore("global", () => {
     checkUpdateAtStart,
     externalControlled,
     // in-memory storage
+    screenStream,
+    rotation,
+    maskSizeW,
+    maskSizeH,
     screenSizeW,
     screenSizeH,
     keyInputFlag,