Merge pull request #48 from AkiChase/40-feature-add-support-for-screenstream

Add support for screenstream

src/components/Device.vue

@@ -44,6 +44,7 @@ import { shutdown } from "../frontcommand/scrcpyMaskCmd";
 import { useGlobalStore } from "../store/global";
 import { useI18n } from "vue-i18n";
 import { closeExternalControl, connectExternalControl } from "../websocket";
+import { LogicalSize, getCurrent } from "@tauri-apps/api/window";
 
 const { t } = useI18n();
 const dialog = useDialog();
@@ -79,7 +80,19 @@ onMounted(async () => {
           } else {
             store.screenSizeW = payload.width;
             store.screenSizeH = payload.height;
-            message.info(t("pages.Device.deviceRotation"));
+            message.info(t("pages.Device.rotation", [payload.rotation * 90]));
+          }
+          if (store.rotation.enable) {
+            let maskW: number;
+            let maskH: number;
+            if (payload.width >= payload.height) {
+              maskW = Math.round(store.rotation.horizontalLength);
+              maskH = Math.round(maskW * (payload.height / payload.width));
+            } else {
+              maskH = Math.round(store.rotation.verticalLength);
+              maskW = Math.round(maskH * (payload.width / payload.height));
+            }
+            getCurrent().setSize(new LogicalSize(maskW + 70, maskH + 30));
           }
           break;
         default:

src/components/Header.vue

@@ -49,6 +49,7 @@ async function maximizeOrRestore() {
   justify-content: end;
   align-items: center;
   border-radius: 0 10px 0 0;
+  user-select: none;
 
   .n-button-group{
     flex-shrink: 0;

src/components/Mask.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import { h, onActivated, onMounted } from "vue";
+import { h, onActivated, onMounted, ref } from "vue";
 import { MessageReactive, NDialog, useDialog, useMessage } from "naive-ui";
 import { useGlobalStore } from "../store/global";
 import { onBeforeRouteLeave, useRouter } from "vue-router";
@@ -11,6 +11,7 @@ import {
   unlistenToEvent,
 } from "../hotkey";
 import { KeyMappingConfig, KeySteeringWheel } from "../keyMappingConfig";
+import ScreenStream from "./ScreenStream.vue";
 import { getVersion } from "@tauri-apps/api/app";
 import { fetch } from "@tauri-apps/plugin-http";
 import { open } from "@tauri-apps/plugin-shell";
@@ -25,7 +26,11 @@ const router = useRouter();
 const message = useMessage();
 const dialog = useDialog();
 
+const curPageActive = ref(false);
+const screenStreamClientId = genClientId();
+
 onBeforeRouteLeave(() => {
+  curPageActive.value = false;
   if (store.controledDevice) {
     unlistenToEvent();
     clearShortcuts();
@@ -34,13 +39,12 @@ onBeforeRouteLeave(() => {
 });
 
 onActivated(async () => {
+  curPageActive.value = true;
   cleanAfterimage();
-  const maskElement = document.getElementById("maskElement") as HTMLElement;
 
   if (store.controledDevice) {
     if (
       applyShortcuts(
-        maskElement,
         store.keyMappingConfigList[store.curKeyMappingIndex],
         store,
         message,
@@ -61,6 +65,13 @@ onMounted(async () => {
   store.checkAdb = checkAdb;
   setTimeout(() => {
     checkAdb();
+    // listen to window resize event
+    const maskElement = document.getElementById("maskElement") as HTMLElement;
+    const appWindow = getCurrent();
+    appWindow.onResized(() => {
+      store.maskSizeH = maskElement.clientHeight;
+      store.maskSizeW = maskElement.clientWidth;
+    });
   }, 500);
 });
 
@@ -79,6 +90,17 @@ async function checkAdb() {
   }
 }
 
+function genClientId() {
+  let result = "";
+  const characters =
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+  const charactersLength = characters.length;
+  for (let i = 0; i < 16; i++) {
+    result += characters.charAt(Math.floor(Math.random() * charactersLength));
+  }
+  return result;
+}
+
 async function loadLocalStore() {
   const localStore = new Store("store.bin");
   // loading keyMappingConfigList from local store
@@ -135,8 +157,25 @@ async function loadLocalStore() {
   };
 
   // loading checkUpdateAtStart from local store
-  let checkUpdateAtStart = await localStore.get<boolean>("checkUpdateAtStart");
+  const checkUpdateAtStart = await localStore.get<boolean>(
+    "checkUpdateAtStart"
+  );
   store.checkUpdateAtStart = checkUpdateAtStart ?? true;
+
+  // loading rotation from local store
+  const rotation = await localStore.get<{
+    enable: boolean;
+    verticalLength: number;
+    horizontalLength: number;
+  }>("rotation");
+  if (rotation) store.rotation = rotation;
+
+  // loading screenStream from local store
+  const screenStream = await localStore.get<{
+    enable: boolean;
+    address: string;
+  }>("screenStream");
+  if (screenStream) store.screenStream = screenStream;
 }
 
 async function cleanAfterimage() {
@@ -226,6 +265,10 @@ async function checkUpdate() {
   </div>
   <template v-if="store.keyMappingConfigList.length">
     <div @contextmenu.prevent class="mask" id="maskElement"></div>
+    <ScreenStream
+      :cid="screenStreamClientId"
+      v-if="curPageActive && store.controledDevice && store.screenStream.enable"
+    />
     <div
       v-if="store.maskButton.show"
       :style="'--transparency: ' + store.maskButton.transparency"
@@ -344,4 +387,3 @@ async function checkUpdate() {
   }
 }
 </style>
-h, import { getVersion } from "@tauri-apps/api/app";

src/components/ScreenStream.vue

@@ -0,0 +1,75 @@
+<script setup lang="ts">
+import { onBeforeUnmount, onMounted, ref } from "vue";
+import { useGlobalStore } from "../store/global";
+import { MessageReactive, useMessage } from "naive-ui";
+import { ScreenStream } from "../screenStream";
+
+const props = defineProps<{
+  cid: string;
+}>();
+
+const store = useGlobalStore();
+const message = useMessage();
+
+const streamImg = ref<HTMLImageElement | null>(null);
+
+let msgReactive: MessageReactive | null = null;
+
+function connectScreenStream() {
+  if (streamImg.value) {
+    const ss = new ScreenStream(streamImg.value, props.cid);
+    ss.connect(
+      store.screenStream.address,
+      () => {},
+      () => {
+        msgReactive = message.error("投屏连接失败。关闭此信息将尝试重新连接", {
+          duration: 0,
+          closable: true,
+          onClose: () => connectScreenStream(),
+        });
+      }
+    );
+  }
+}
+
+onMounted(() => {
+  connectScreenStream();
+});
+
+onBeforeUnmount(() => {
+  if (streamImg.value) streamImg.value.src = "";
+  if (msgReactive) {
+    msgReactive.destroy();
+  }
+});
+</script>
+
+<template>
+  <div class="screen-stream">
+    <img
+      :style="{
+        width: `${store.maskSizeW}px`,
+        height: `${store.maskSizeH}px`,
+      }"
+      ref="streamImg"
+    />
+  </div>
+</template>
+
+<style scoped lang="scss">
+.screen-stream {
+  position: absolute;
+  left: 70px;
+  top: 30px;
+  height: 100%;
+  width: 100%;
+  z-index: 0;
+  pointer-events: none;
+  user-select: none;
+
+  img {
+    pointer-events: none;
+    user-select: none;
+  }
+}
+</style>

src/components/setting/Mask.vue

@@ -2,7 +2,6 @@
 import { onMounted, onUnmounted, ref } from "vue";
 import {
   NH4,
-  NP,
   NForm,
   NGrid,
   NFormItemGi,
@@ -16,6 +15,7 @@ import {
   NSlider,
   NFormItem,
   NCheckbox,
+  NInput,
 } from "naive-ui";
 import {
   LogicalPosition,
@@ -234,8 +234,62 @@ onUnmounted(() => {
           />
         </NFormItemGi>
       </NGrid>
-      <NP>{{ $t("pages.Setting.Mask.areaTip") }}</NP>
     </NForm>
+
+    <NH4 prefix="bar">{{ $t("pages.Setting.Mask.rotation.title") }}</NH4>
+    <NFormItem
+      :label="$t('pages.Setting.Mask.rotation.rotateWithDevice')"
+      label-placement="left"
+    >
+      <NCheckbox
+        v-model:checked="store.rotation.enable"
+        @update:checked="localStore.set('rotation', store.rotation)"
+      />
+    </NFormItem>
+    <NGrid :cols="2">
+      <NFormItemGi
+        :label="$t('pages.Setting.Mask.rotation.verticalLength')"
+        label-placement="left"
+      >
+        <NInputNumber
+          v-model:value="store.rotation.verticalLength"
+          @update:value="localStore.set('rotation', store.rotation)"
+          :placeholder="$t('pages.Setting.Mask.rotation.verticalLength')"
+        />
+      </NFormItemGi>
+      <NFormItemGi
+        :label="$t('pages.Setting.Mask.rotation.horizontalLength')"
+        label-placement="left"
+      >
+        <NInputNumber
+          v-model:value="store.rotation.horizontalLength"
+          @update:value="localStore.set('rotation', store.rotation)"
+          :placeholder="$t('pages.Setting.Mask.rotation.horizontalLength')"
+        />
+      </NFormItemGi>
+    </NGrid>
+
+    <NH4 prefix="bar">ScreenStream</NH4>
+    <NFormItem
+      :label="$t('pages.Setting.Mask.screenStream.enable')"
+      label-placement="left"
+    >
+      <NCheckbox
+        v-model:checked="store.screenStream.enable"
+        @update:checked="localStore.set('screenStream', store.screenStream)"
+      />
+    </NFormItem>
+    <NFormItem
+      :label="$t('pages.Setting.Mask.screenStream.address')"
+      label-placement="left"
+    >
+      <NInput
+        v-model:value="store.screenStream.address"
+        @update:value="localStore.set('screenStream', store.screenStream)"
+        clearable
+        :placeholder="$t('pages.Setting.Mask.screenStream.addressPlaceholder')"
+      />
+    </NFormItem>
   </div>
 </template>
 

src/hotkey.ts

@@ -24,7 +24,6 @@ import {
 import { useGlobalStore } from "./store/global";
 import { LogicalPosition, getCurrent } from "@tauri-apps/api/window";
 import { useI18n } from "vue-i18n";
-import { UnlistenFn } from "@tauri-apps/api/event";
 import { KeyToCodeMap } from "./frontcommand/KeyToCodeMap";
 import {
   AndroidKeyEventAction,
@@ -36,13 +35,13 @@ import { sendInjectKeycode } from "./frontcommand/controlMsg";
 function clientxToPosx(clientx: number) {
   return clientx < 70
     ? 0
-    : Math.floor((clientx - 70) * (store.screenSizeW / maskSizeW));
+    : Math.floor((clientx - 70) * (store.screenSizeW / store.maskSizeW));
 }
 
 function clientyToPosy(clienty: number) {
   return clienty < 30
     ? 0
-    : Math.floor((clienty - 30) * (store.screenSizeH / maskSizeH));
+    : Math.floor((clienty - 30) * (store.screenSizeH / store.maskSizeH));
 }
 
 function clientxToPosOffsetx(clientx: number, posx: number, scale = 1) {
@@ -59,6 +58,9 @@ function clientPosToSkillOffset(
   clientPos: { x: number; y: number },
   range: number
 ): { offsetX: number; offsetY: number } {
+  const maskSizeH = store.maskSizeH;
+  const maskSizeW = store.maskSizeW;
+
   const maxLength = (120 / maskSizeH) * store.screenSizeH;
   const centerX = maskSizeW * 0.5;
   const centerY = maskSizeH * 0.5;
@@ -1511,40 +1513,24 @@ export function clearShortcuts() {
   loopDownKeyCBMap.clear();
   upKeyCBMap.clear();
   cancelAbleKeyList.length = 0;
-
-  // unlisten to resize
-  unlistenResize();
 }
 
 export function applyShortcuts(
-  element: HTMLElement,
   keyMappingConfig: KeyMappingConfig,
   globalStore: ReturnType<typeof useGlobalStore>,
   messageAPI: ReturnType<typeof useMessage>,
   i18nT: ReturnType<typeof useI18n>["t"]
 ) {
   store = globalStore;
-  maskElement = element;
+  maskElement = document.getElementById("maskElement") as HTMLElement;
   message = messageAPI;
   t = i18nT;
 
-  maskSizeW = maskElement.clientWidth;
-  maskSizeH = maskElement.clientHeight;
-  // listen to resize to update mask size
-  getCurrent()
-    .onResized(() => {
-      maskSizeW = maskElement.clientWidth;
-      maskSizeH = maskElement.clientHeight;
-    })
-    .then((f) => (unlistenResize = f));
-
   addClickShortcuts("M0", 0);
 
   return applyKeyMappingConfigShortcuts(keyMappingConfig);
 }
 
-let maskSizeW: number;
-let maskSizeH: number;
 let mouseX = 0;
 let mouseY = 0;
 let store: ReturnType<typeof useGlobalStore>;
@@ -1552,8 +1538,6 @@ let maskElement: HTMLElement;
 let message: ReturnType<typeof useMessage>;
 let t: ReturnType<typeof useI18n>["t"];
 
-let unlistenResize: UnlistenFn;
-
 const downKeyMap: Map<string, boolean> = new Map();
 const downKeyCBMap: Map<string, () => Promise<void>> = new Map();
 const loopDownKeyCBMap: Map<string, () => Promise<void>> = new Map();

src/i18n/en-US.json

@@ -37,10 +37,9 @@
       "wsConnect": "Control",
       "adbDeviceError": "Unable to get available devices",
       "adbConnectError": "Wireless connection failed",
-      "deviceRotation": "Device rotation"
+      "rotation": "Device rotation {0}°"
     },
     "Mask": {
-      "inputBoxPlaceholder": "Input text and then press enter/esc",
       "keyconfigException": "The key mapping config is abnormal, please delete this config",
       "blankConfig": "Blank config",
       "checkUpdate": {
@@ -77,9 +76,9 @@
         "areaSaved": "Mask area saved",
         "incorrectArea": "Please enter the coordinates and size of the mask correctly",
         "buttonPrompts": "Button prompts",
-        "ifButtonPrompts": "Whether to display",
+        "ifButtonPrompts": "Show key prompts",
         "opacity": "Opacity",
-        "areaAdjust": "Mask adjustment",
+        "areaAdjust": "Mask area",
         "areaPlaceholder": {
           "x": "X coordinate of upper left corner"
         },
@@ -88,7 +87,17 @@
           "w": "Mask width",
           "h": "Mask height"
         },
-        "areaTip": "Tip: The mask size and device size will be used for coordinate conversion, please ensure the accuracy of the size"
+        "rotation": {
+          "title": "Device rotation",
+          "rotateWithDevice": "Follow device rotation",
+          "verticalLength": "Mask height in vertical screen",
+          "horizontalLength": "Mask width in horizontal screen "
+        },
+        "screenStream": {
+          "enable": "Enable mirror",
+          "address": "Screen mirror address",
+          "addressPlaceholder": "Please enter the ScreenStream screen mirror address"
+        }
       },
       "Basic": {
         "delLocalStore": {

src/i18n/zh-CN.json

@@ -37,7 +37,7 @@
       "wsConnect": "控制",
       "adbDeviceError": "无法获取可用设备",
       "adbConnectError": "无线连接失败",
-      "deviceRotation": "设备旋转"
+      "rotation": "设备旋转 {0}°"
     },
     "Mask": {
       "keyconfigException": "按键方案异常，请删除此方案",
@@ -56,7 +56,6 @@
         "content": "请前往设备页面，控制任意设备",
         "positiveText": "去控制"
       },
-      "inputBoxPlaceholder": "输入文本后按Enter/Esc",
       "sightMode": "鼠标已锁定, 按 {0} 键解锁",
       "checkAdb": "adb不可用，软件无法正常运行，请确保系统已安装adb，并正确添加到了Path环境变量中: {0}",
       "keyInputMode": "已进入按键输入模式，关闭本消息可退出"
@@ -71,9 +70,9 @@
         "incorrectArea": "请正确输入蒙版的坐标和尺寸",
         "areaSaved": "蒙版区域已保存",
         "buttonPrompts": "按键提示",
-        "ifButtonPrompts": "是否显示",
+        "ifButtonPrompts": "显示按键提示",
         "opacity": "不透明度",
-        "areaAdjust": "蒙版调整",
+        "areaAdjust": "蒙版区域",
         "areaPlaceholder": {
           "x": "左上角X坐标"
         },
@@ -82,12 +81,22 @@
           "w": "蒙版宽度",
           "h": "蒙版高度"
         },
-        "areaTip": "提示：蒙版尺寸与设备尺寸将用于坐标转换，请保证尺寸的准确性",
         "areaFormMissing": {
           "x": "请输入蒙版左上角X坐标",
           "y": "请输入蒙版左上角Y坐标",
           "w": "请输入蒙版宽度",
           "h": "请输入蒙版高度"
+        },
+        "rotation": {
+          "title": "设备旋转",
+          "rotateWithDevice": "跟随设备旋转",
+          "verticalLength": "竖屏蒙版高度",
+          "horizontalLength": "横屏蒙版宽度"
+        },
+        "screenStream": {
+          "enable": "启用投屏",
+          "address": "投屏地址",
+          "addressPlaceholder": "请输入 ScreenStream 投屏地址"
         }
       },
       "Basic": {

src/screenStream.ts

@@ -0,0 +1,40 @@
+export class ScreenStream {
+  img: HTMLImageElement;
+  clientId: string;
+  connectTimeoutId: number | undefined;
+
+  public constructor(imgElement: HTMLImageElement, clientId: string) {
+    this.img = imgElement;
+    this.clientId = clientId;
+  }
+
+  public connect(address: string, onConnect: () => void, onError: () => void) {
+    if (address.endsWith("/")) address = address.slice(0, -1);
+    const that = this;
+    const img = that.img;
+    const url = `${address}/stream.mjpeg?clientId=${this.clientId}`;
+
+    img.src = "";
+    clearTimeout(that.connectTimeoutId);
+    new Promise<void>((resolve, reject) => {
+      img.onload = function () {
+        img.onload = null;
+        img.onerror = null;
+        resolve();
+      };
+      img.onerror = function (e) {
+        img.onerror = null;
+        img.onload = null;
+        reject(e);
+      };
+      img.src = url;
+    })
+      .then(() => {
+        onConnect();
+      })
+      .catch(() => {
+        img.src = "";
+        onError();
+      });
+  }
+}

src/store/global.ts

@@ -75,6 +75,20 @@ export const useGlobalStore = defineStore("global", () => {
 
   const keyInputFlag = ref(false);
 
+  const maskSizeW: Ref<number> = ref(0);
+  const maskSizeH: Ref<number> = ref(0);
+
+  const screenStream = ref({
+    enable: false,
+    address: "",
+  });
+
+  const rotation = ref({
+    enable: true,
+    verticalLength: 600,
+    horizontalLength: 800,
+  });
+
   // persistent storage
   const keyMappingConfigList: Ref<KeyMappingConfig[]> = ref([]);
   const curKeyMappingIndex = ref(0);
@@ -92,6 +106,10 @@ export const useGlobalStore = defineStore("global", () => {
     checkUpdateAtStart,
     externalControlled,
     // in-memory storage
+    screenStream,
+    rotation,
+    maskSizeW,
+    maskSizeH,
     screenSizeW,
     screenSizeH,
     keyInputFlag,