Merge pull request #1 from AkiChase/dev

Complete the construction of the basic mouse and key mapping

README.md

@@ -0,0 +1,16 @@
+# Scrcpy-mask
+
+A Scrcpy (control) client in Rust & Tarui aimed at providing mouse and key mapping.
+
+Due to the delay and blurred image quality of the mirror screen. This project found another way, directly abandoned the mirror screen, and instead used a transparent mask to display the screen content behind the window, which fundamentally put an end to the delay in casting the screen.
+
+## Features
+
+- [x] Start scrcpy-server and connect to it
+- [ ] Mouse and keyboard mapping （Partially completed）
+- [ ] Visually setting the mapping
+- [ ] Other setting
+
+## contribution.
+
+If you are interested in this project, you are welcome to submit pull request or issue. But my time and energy is limited, so I may not be able to deal with it all.

package.json

@@ -10,21 +10,22 @@
     "tauri": "tauri"
   },
   "dependencies": {
+    "@tauri-apps/api": ">=2.0.0-beta.0",
+    "@tauri-apps/plugin-os": "2.0.0-beta.2",
+    "@tauri-apps/plugin-shell": ">=2.0.0-beta.0",
     "pinia": "^2.1.7",
     "vue": "^3.3.4",
-    "vue-router": "4",
+    "vue-router": "4"
-    "@tauri-apps/api": ">=2.0.0-beta.0",
-    "@tauri-apps/plugin-shell": ">=2.0.0-beta.0"
   },
   "devDependencies": {
-    "@vitejs/plugin-vue": "^5.0.4",
-    "typescript": "^5.0.2",
-    "vite": "^5.0.0",
-    "vue-tsc": "^1.8.5",
     "@tauri-apps/cli": ">=2.0.0-beta.0",
     "@vicons/fluent": "^0.12.0",
     "@vicons/ionicons5": "^0.12.0",
+    "@vitejs/plugin-vue": "^5.0.4",
     "naive-ui": "^2.38.1",
-    "sass": "^1.71.1"
+    "sass": "^1.71.1",
+    "typescript": "^5.0.2",
+    "vite": "^5.0.0",
+    "vue-tsc": "^1.8.5"
   }
 }

src-tauri/Cargo.toml

@@ -2,7 +2,7 @@
 name = "scrcpy-mask"
 version = "0.0.0"
 description = "A Tauri App"
-authors = ["you"]
+authors = ["AkiChase"]
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -12,7 +12,7 @@ tauri-build = { version = "2.0.0-beta", features = [] }
 
 [dependencies]
 tauri = { version = "2.0.0-beta", features = ["macos-private-api"] }
-tauri-plugin-shell = "2.0.0-beta"
+tauri-plugin-os = "2.0.0-beta"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 anyhow = "1.0"

src-tauri/capabilities/default.json

@@ -13,6 +13,8 @@
     "window:allow-close",
     "window:allow-is-maximizable",
     "window:allow-start-dragging",
-    "window:allow-unmaximize"
+    "window:allow-unmaximize",
+    "os:default",
+    "os:allow-platform"
   ]
 }

src-tauri/src/adb.rs

@@ -5,6 +5,9 @@ use std::{
     process::{Child, Command, Stdio},
 };
 
+#[cfg(target_os = "windows")]
+use std::os::windows::process::CommandExt;
+
 use anyhow::{Context, Ok, Result};
 
 #[derive(Clone, Debug, serde::Serialize)]
@@ -89,8 +92,15 @@ pub struct Adb;
 /// But some output of command won't be output, like adb service startup information.
 impl Adb {
     fn cmd_base(res_dir: &PathBuf) -> Command {
+        #[cfg(target_os = "windows")]{
+            let mut cmd = Command::new(ResHelper::get_file_path(res_dir, ResourceName::Adb));
+            cmd.creation_flags(0x08000000); // CREATE_NO_WINDOW
+            cmd
+        }
+        #[cfg(not(target_os = "windows"))]{
             Command::new(ResHelper::get_file_path(res_dir, ResourceName::Adb))
         }
+    }
 
     /// execute "adb devices" and return devices list
     pub fn cmd_devices(res_dir: &PathBuf) -> Result<Vec<Device>> {

src-tauri/src/main.rs

@@ -96,7 +96,7 @@ fn start_scrcpy_server(
         let share_app = app.clone();
         let listen_handler = share_app.listen("front-command", move |event| {
             let sender = front_msg_sender.clone();
-            println!("收到front-command: {}", event.payload());
+            // println!("收到front-command: {}", event.payload());
             tokio::spawn(async move {
                 if let Err(e) = sender.send(event.payload().into()).await {
                     println!("front-command转发失败: {}", e);
@@ -130,9 +130,44 @@ async fn main() {
                     .join("resource"),
             )
             .unwrap();
+
+            let main_window = app.get_webview_window("main").unwrap();
+
+            #[cfg(windows)]
+            {
+                let scale_factor = main_window.scale_factor().unwrap();
+                main_window
+                    .set_size(tauri::Size::Physical(tauri::PhysicalSize {
+                        width: 1350,
+                        height: 750,
+                    }))
+                    .unwrap();
+
+                main_window
+                    .with_webview(move |webview| {
+                        unsafe {
+                            // see https://docs.rs/webview2-com/0.19.1/webview2_com/Microsoft/Web/WebView2/Win32/struct.ICoreWebView2Controller.html
+                            webview
+                                .controller()
+                                .SetZoomFactor(1.0 / scale_factor)
+                                .unwrap();
+                        }
+                    })
+                    .unwrap();
+            }
+            #[cfg(target_os = "macos")]
+            {
+                main_window
+                    .set_size(tauri::Size::Logical(tauri::LogicalSize {
+                        width: 1350.,
+                        height: 750.,
+                    }))
+                    .unwrap();
+            }
+
             Ok(())
         })
-        .plugin(tauri_plugin_shell::init())
+        .plugin(tauri_plugin_os::init())
         .invoke_handler(tauri::generate_handler![
             adb_devices,
             get_screen_size,

src-tauri/src/socket.rs

@@ -173,7 +173,6 @@ async fn recv_front_msg(
                             &mut write_half,
                         )
                         .await;
-                        println!("控制信息发送完成！");
                         continue;
                     } else {
                         // 处理Scrcpy Mask命令

src-tauri/tauri.conf.json

@@ -1,7 +1,7 @@
 {
   "productName": "scrcpy-mask",
   "version": "0.0.0",
-  "identifier": "com.tauri.dev",
+  "identifier": "com.akichase.mask",
   "build": {
     "beforeDevCommand": "pnpm dev",
     "devUrl": "http://localhost:1420",
@@ -25,10 +25,12 @@
     "active": true,
     "targets": "all",
     "icon": [
-      "icons/icon.icns"
+      "icons/32x32.png",
+      "icons/128x128.png",
+      "icons/128x128@2x.png",
+      "icons/icon.icns",
+      "icons/icon.ico"
     ],
-    "resources":[
+    "resources": ["resource/*"]
-      "resource/*"
-    ]
   }
 }

src/components/Device.vue

@@ -15,6 +15,7 @@ import {
   pushServerFile,
   forwardServerPort,
   startScrcpyServer,
+  getScreenSize,
 } from "../invoke";
 import {
   NH4,
@@ -43,9 +44,7 @@ const store = useGlobalStore();
 const port = ref(27183);
 
 //#region listener
-const deviceWaitForMetadataTask: ((
+const deviceWaitForMetadataTask: ((deviceName: string) => void)[] = [];
-  deviceName: string
-) => void)[] = [];
 
 let unlisten: UnlistenFn | undefined;
 onMounted(async () => {
@@ -175,6 +174,8 @@ async function onMenuSelect(key: string) {
         "00000000" + Math.floor(Math.random() * 100000).toString(16)
       ).slice(-8);
 
+      let screenSize = await getScreenSize(device.id);
+
       await pushServerFile(device.id);
       await forwardServerPort(device.id, scid, port.value);
       await startScrcpyServer(device.id, scid, `127.0.0.1:${port.value}`);
@@ -185,7 +186,8 @@ async function onMenuSelect(key: string) {
           scid,
           deviceName,
           device,
-        }
+          screenSize,
+        };
         nextTick(() => {
           store.hideLoading();
         });
@@ -200,6 +202,13 @@ async function refreshDevices() {
   devices.value = await adbDevices();
   store.hideLoading();
 }
+
+const screenSizeInfo = computed(() => {
+  if (store.controledDevice) {
+    return `${store.controledDevice.screenSize[0]} x ${store.controledDevice.screenSize[1]}`;
+  }
+  return "";
+});
 </script>
 
 <template>
@@ -221,7 +230,11 @@ async function refreshDevices() {
           v-if="!store.controledDevice"
         />
         <div class="controled-device" v-if="store.controledDevice">
-          <div>{{ store.controledDevice.deviceName }} ({{ store.controledDevice.device.id }})</div>
+          <div>
+            {{ store.controledDevice.deviceName }} ({{
+              store.controledDevice.device.id
+            }})
+          </div>
           <div class="device-op">
             <NTooltip trigger="hover">
               <template #trigger>
@@ -231,16 +244,11 @@ async function refreshDevices() {
                   </template>
                 </NButton>
               </template>
-              scid: {{ store.controledDevice.scid }}
+              scid: {{ store.controledDevice.scid }} <br />status:
-              <br />status: {{ store.controledDevice.device.status }}
+              {{ store.controledDevice.device.status }} <br />screen:
+              {{ screenSizeInfo }}
             </NTooltip>
-
+            <NButton quaternary circle type="error" @click="shutdownSC()">
-            <NButton
-              quaternary
-              circle
-              type="error"
-              @click="shutdownSC()"
-            >
               <template #icon>
                 <NIcon><CloseCircle /></NIcon>
               </template>

src/components/Header.vue

@@ -15,7 +15,7 @@ async function maximizeOrRestore() {
 <template>
   <div data-tauri-drag-region class="header">
     <NButtonGroup>
-      <NButton quaternary :focusable="false" @click="appWindow.minimize()">
+      <NButton quaternary :focusable="false" @click="getCurrent().minimize()">
         <template #icon>
           <NIcon><Subtract16Regular /></NIcon>
         </template>

src/components/Mask.vue

@@ -2,10 +2,12 @@
 import { onActivated, ref } from "vue";
 import { NDialog } from "naive-ui";
 import { useGlobalStore } from "../store/global";
-import { useRouter } from "vue-router";
+import { onBeforeRouteLeave, useRouter } from "vue-router";
-import { getCurrent } from "@tauri-apps/api/window";
+import {
-import { initShortcuts } from "../hotkey";
+  initShortcuts,
-import { getScreenSize } from "../invoke";
+  listenToKeyEvent,
+  unlistenToKeyEvent,
+} from "../hotkey";
 
 const maskRef = ref<HTMLElement | null>(null);
 
@@ -14,20 +16,26 @@ const router = useRouter();
 
 let isShortcutInited = false;
 
+onBeforeRouteLeave(() => {
+  if (isShortcutInited) {
+    if (maskRef.value) {
+      unlistenToKeyEvent();
+    }
+  }
+});
+
 onActivated(async () => {
   if (isShortcutInited) {
-    maskRef.value?.focus();
+    if (maskRef.value) {
+      listenToKeyEvent();
+    }
     return;
   }
   if (store.controledDevice) {
-    let screenSize = await getScreenSize(store.controledDevice.device.id);
     if (maskRef.value) {
-      const appWindow = getCurrent();
+      initShortcuts(store.controledDevice.screenSize, maskRef.value);
-      let posFactor = await appWindow.scaleFactor();
+      listenToKeyEvent();
-      initShortcuts(maskRef.value, posFactor, screenSize);
       isShortcutInited = true;
-      maskRef.value.focus();
-      console.log("热键已载入");
     }
   }
 });
@@ -36,7 +44,6 @@ function toStartServer() {
   router.replace({ name: "device" });
 }
 
-// TODO 监听快捷键
 // TODO 按键设置
 // TODO 渲染按钮
 </script>
@@ -56,7 +63,7 @@ function toStartServer() {
   </div>
   <div
     v-show="store.controledDevice"
-    tabindex="-1"
+    @contextmenu.prevent
     class="mask"
     ref="maskRef"
   ></div>
@@ -66,11 +73,7 @@ function toStartServer() {
 .mask {
   background-color: rgba(255, 255, 255, 0.2);
   overflow: hidden;
-
+  cursor: pointer;
-  &:focus {
-    outline: none;
-    box-shadow: 0 0 5px var(--primary-color);
-  }
 }
 .notice {
   background-color: rgba(255, 255, 255, 0.2);

src/components/Sidebar.vue

@@ -10,14 +10,47 @@ import {
 } from "@vicons/ionicons5";
 import { Keyboard24Regular } from "@vicons/fluent";
 import { NIcon } from "naive-ui";
+import { useGlobalStore } from "../store/global";
+import { sendInjectKeycode } from "../frontcommand/controlMsg";
+import {
+  AndroidKeyEventAction,
+  AndroidKeycode,
+  AndroidMetastate,
+} from "../frontcommand/android";
 
 const router = useRouter();
 const route = useRoute();
+const store = useGlobalStore();
 
 function nav(name: string) {
   router.replace({ name });
 }
 
+function sleep(time: number) {
+  return new Promise<void>((resolve) => {
+    setTimeout(() => {
+      resolve();
+    }, time);
+  });
+}
+
+async function sendKeyCodeToDevice(code: AndroidKeycode) {
+  if (store.controledDevice) {
+    await sendInjectKeycode({
+      action: AndroidKeyEventAction.AKEY_EVENT_ACTION_DOWN,
+      keycode: code,
+      repeat: 0,
+      metastate: AndroidMetastate.AMETA_NONE,
+    });
+    await sleep(50);
+    await sendInjectKeycode({
+      action: AndroidKeyEventAction.AKEY_EVENT_ACTION_UP,
+      keycode: code,
+      repeat: 0,
+      metastate: AndroidMetastate.AMETA_NONE,
+    });
+  }
+}
 </script>
 
 <template>
@@ -50,17 +83,17 @@ function nav(name: string) {
     </div>
 
     <div class="nav">
-      <div>
+      <div @click="sendKeyCodeToDevice(AndroidKeycode.AKEYCODE_BACK)">
         <NIcon>
           <ReturnDownBackOutline />
         </NIcon>
       </div>
-      <div>
+      <div @click="sendKeyCodeToDevice(AndroidKeycode.AKEYCODE_HOME)">
         <NIcon>
           <StopOutline />
         </NIcon>
       </div>
-      <div>
+      <div @click="sendKeyCodeToDevice(AndroidKeycode.AKEYCODE_APP_SWITCH)">
         <NIcon>
           <ListOutline />
         </NIcon>
@@ -79,6 +112,7 @@ function nav(name: string) {
   flex-direction: column;
   justify-content: space-between;
   user-select: none;
+  -webkit-user-select: none;
 
   .logo {
     height: 30px;
@@ -88,6 +122,7 @@ function nav(name: string) {
     justify-content: center;
     align-items: center;
     color: var(--light-color);
+    cursor: pointer;
   }
 
   .module {

src/components/keyboard/KeyBoard.vue

@@ -1,6 +1,5 @@
 <script setup lang="ts">
-import { getCurrent } from "@tauri-apps/api/window";
+import { Ref, onActivated, ref } from "vue";
-import { onActivated, onMounted, ref } from "vue";
 import { onBeforeRouteLeave } from "vue-router";
 
 // TODO 添加右侧按键列表用于拖放
@@ -10,13 +9,12 @@ const keyboardElement = ref<HTMLElement | null>(null);
 const mouseX = ref(0);
 const mouseY = ref(0);
 
-let posFactor = 1;
 function clientxToPosx(clientx: number) {
-  return clientx < 70 ? 0 : Math.floor((clientx - 70) * posFactor);
+  return clientx < 70 ? 0 : Math.floor(clientx - 70);
 }
 
 function clientyToPosy(clienty: number) {
-  return clienty < 30 ? 0 : Math.floor((clienty - 30) * posFactor);
+  return clienty < 30 ? 0 : Math.floor(clienty - 30);
 }
 
 let ignoreMousemove = true;
@@ -27,17 +25,23 @@ function mousemoveHandler(event: MouseEvent) {
   mouseY.value = clientyToPosy(event.clientY);
 }
 
-onMounted(async () => {
+const keyboardCodeList: Ref<string[]> = ref([]);
-  const appWindow = getCurrent();
+function keyupHandler(event: KeyboardEvent) {
-  posFactor = await appWindow.scaleFactor();
+  event.preventDefault();
-});
+  if (keyboardCodeList.value.length > 10) {
+    keyboardCodeList.value.shift();
+    keyboardCodeList.value.push(event.code);
+  } else keyboardCodeList.value.push(event.code);
+}
 
 onActivated(() => {
   keyboardElement.value?.addEventListener("mousemove", mousemoveHandler);
+  document.addEventListener("keyup", keyupHandler);
 });
 
 onBeforeRouteLeave(() => {
   keyboardElement.value?.removeEventListener("mousemove", mousemoveHandler);
+  document.removeEventListener("keyup", keyupHandler);
 });
 </script>
 
@@ -45,6 +49,9 @@ onBeforeRouteLeave(() => {
   <div ref="keyboardElement" class="keyboard">
     此处最好用其他颜色的蒙版，和右侧的按键列表区同色
     <div>{{ mouseX }}, {{ mouseY }}</div>
+    <div v-for="code in keyboardCodeList">
+      {{ code }}
+    </div>
   </div>
 </template>
 

src/components/setting/Mask.vue

@@ -12,32 +12,49 @@ import {
   NIcon,
   FormInst,
   useMessage,
+  NP,
 } from "naive-ui";
 import {
   PhysicalPosition,
   PhysicalSize,
   getCurrent,
 } from "@tauri-apps/api/window";
+import { platform } from "@tauri-apps/plugin-os";
 import { SettingsOutline } from "@vicons/ionicons5";
 import { UnlistenFn } from "@tauri-apps/api/event";
 
 let unlistenResize: UnlistenFn = () => {};
 let unlistenMove: UnlistenFn = () => {};
 
-async function refreshAreaModel(size?: PhysicalSize, pos?: PhysicalPosition) {
+let factor = 1;
-  const appWindow = getCurrent();
+let platformName = "";
-  const factor = await appWindow.scaleFactor();
-  // header size and sidebar size
-  const mt = 30 * factor;
-  const ml = 70 * factor;
 
+// macos: use logical position and size to refresh the area model
+// others: use pyhsical position and size to refresh the area model
+async function refreshAreaModel(size?: PhysicalSize, pos?: PhysicalPosition) {
+  // header size and sidebar size
+  const mt = 30;
+  const ml = 70;
+
+  if (platformName === "macos") {
+    // use logical position and size
+    if (size !== undefined) {
+      areaModel.value.sizeW = Math.floor((size.width - ml) / factor);
+      areaModel.value.sizeH = Math.floor((size.height - mt) / factor);
+    }
+    if (pos !== undefined) {
+      areaModel.value.posX = Math.floor((pos.x + ml) / factor);
+      areaModel.value.posY = Math.floor((pos.y + mt) / factor);
+    }
+  } else {
+    if (size !== undefined) {
+      areaModel.value.sizeW = Math.floor(size.width - ml);
+      areaModel.value.sizeH = Math.floor(size.height - mt);
+    }
     if (pos !== undefined) {
       areaModel.value.posX = Math.floor(pos.x + ml);
       areaModel.value.posY = Math.floor(pos.y + mt);
     }
-  if (size !== undefined) {
-    areaModel.value.sizeW = Math.floor(size.width - ml);
-    areaModel.value.sizeH = Math.floor(size.height - mt);
   }
 }
 
@@ -45,6 +62,7 @@ const message = useMessage();
 
 const formRef = ref<FormInst | null>(null);
 
+// logical pos and size of the mask area
 const areaModel = ref({
   posX: 0,
   posY: 0,
@@ -92,6 +110,7 @@ function handleAdjustClick(e: MouseEvent) {
   });
 }
 
+// TODO 等待官方合并修复分支后检查表现是否正常
 // move and resize window to the selected window (control) area
 async function adjustMaskArea() {
   // header size and sidebar size
@@ -99,32 +118,32 @@ async function adjustMaskArea() {
   const ml = 70;
 
   const appWindow = getCurrent();
-  const factor = await appWindow.scaleFactor();
 
   const pos = new PhysicalPosition(
-    areaModel.value.posX,
+    areaModel.value.posX - ml,
-    areaModel.value.posY
+    areaModel.value.posY - mt
-  ).toLogical(factor);
+  );
-  pos.y -= mt;
-  pos.x -= ml;
-
-  if (pos.x <= 0 || pos.y <= 0) {
-    message.warning("蒙版区域坐标过小，可能导致其他部分不可见");
-  }
 
   const size = new PhysicalSize(
-    areaModel.value.sizeW,
+    areaModel.value.sizeW + ml,
-    areaModel.value.sizeH
+    areaModel.value.sizeH + mt
-  ).toLogical(factor);
+  );
-  size.width += ml;
-  size.height += mt;
 
+  if (platformName === "macos") {
+    // use logical position and size
+    await appWindow.setPosition(pos.toLogical(factor));
+    await appWindow.setSize(size.toLogical(factor));
+  } else {
     await appWindow.setPosition(pos);
     await appWindow.setSize(size);
   }
+}
 
 onMounted(async () => {
   const appWindow = getCurrent();
+  factor = await appWindow.scaleFactor();
+  platformName = await platform();
+
   unlistenResize = await appWindow.onResized(({ payload: size }) => {
     refreshAreaModel(size, undefined);
   });
@@ -194,6 +213,7 @@ onUnmounted(() => {
           />
         </NFormItemGi>
       </NGrid>
+      <NP>提示：使用物理坐标、尺寸</NP>
     </NForm>
   </div>
 </template>

src/store/global.ts

@@ -15,6 +15,7 @@ export const useGlobalStore = defineStore("counter", () => {
     scid: string;
     deviceName: string;
     device: Device;
+    screenSize: [number, number];
   }
 
   const controledDevice: Ref<ControledDevice|null> = ref(null);