Skip to content

Commit

Permalink
Merge pull request #122 from shamanec/handle-landscape
Browse files Browse the repository at this point in the history 
* Basic landscape remote control support
  • Loading branch information
@shamanec
shamanec authored Sep 13, 2024
2 parents 75453be + b2728fb commit 21b9fcc
Show file tree
Hide file tree
Showing 3 changed files with 179 additions and 88 deletions.
3 changes: 2 additions & 1 deletion common/models/models.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,8 @@ type Device struct {
ProviderState string `json:"provider_state" bson:"-"` // current state of the device on the provider - init, preparing, live
/// PROVIDER ONLY VALUES
//// RETURNABLE VALUES
InstalledApps []string `json:"installed_apps" bson:"-"` // list of installed apps on device
InstalledApps []string `json:"installed_apps" bson:"-"` // list of installed apps on device
UsesCustomWDA bool `json:"uses_custom_wda" bson:"-"` // Flag for iOS device if provider sets up custom WDA
///// NON-RETURNABLE VALUES
AppiumSessionID string `json:"-" bson:"-"` // current Appium session ID
WDASessionID string `json:"-" bson:"-"` // current WebDriverAgent session ID
Expand Down
263 changes: 176 additions & 87 deletions hub/gads-ui/src/components/DeviceControl/StreamCanvas.js
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { Auth } from "../../contexts/Auth"
import { useContext, useEffect, useState } from "react"
import './StreamCanvas.css'
import { Button, Divider, Grid, Stack } from "@mui/material"
import { Button, Divider, Grid, Stack, Tooltip } from "@mui/material"
import HomeIcon from '@mui/icons-material/Home';
import LockOpenIcon from '@mui/icons-material/LockOpen';
import LockIcon from '@mui/icons-material/Lock';
Expand All @@ -15,131 +15,184 @@ export default function StreamCanvas({ deviceData }) {
width: 0,
height: 0
});
const [isPortrait, setIsPortrait] = useState(true)
const handleOrientationButtonClick = (isPortrait) => {
setIsPortrait(isPortrait);
}

let deviceX = parseInt(deviceData.screen_width, 10)
let deviceY = parseInt(deviceData.screen_height, 10)
let screen_ratio = deviceX / deviceY
let landscapeScreenRatio = deviceY / deviceX

const streamData = {
udid: deviceData.udid,
deviceX: deviceX,
deviceY: deviceY,
screen_ratio: screen_ratio,
canvasHeight: canvasSize.height,
canvasWidth: canvasSize.width
canvasWidth: canvasSize.width,
isPortrait: isPortrait,
device_os: deviceData.os,
uses_custom_wda: deviceData.uses_custom_wda
}

let streamUrl = ""
if (deviceData.os === 'ios') {
// streamUrl = `http://192.168.68.109:10000/device/${deviceData.udid}/ios-stream-mjpeg`
// streamUrl = `http://192.168.1.6:10000/device/${deviceData.udid}/ios-stream-mjpeg`
streamUrl = `/device/${deviceData.udid}/ios-stream-mjpeg`
} else {
// streamUrl = `http://192.168.68.109:10000/device/${deviceData.udid}/android-stream-mjpeg`
// streamUrl = `http://192.168.1.6:10000/device/${deviceData.udid}/android-stream-mjpeg`
streamUrl = `/device/${deviceData.udid}/android-stream-mjpeg`
}

useEffect(() => {
const updateCanvasSize = () => {
let canvasHeight = window.innerHeight * 0.7
let canvasWidth = canvasHeight * screen_ratio
let canvasWidth, canvasHeight
if (isPortrait) {
canvasHeight = window.innerHeight * 0.7
canvasWidth = canvasHeight * screen_ratio
} else {
canvasWidth = window.innerWidth * 0.4
canvasHeight = canvasWidth / landscapeScreenRatio
}

setCanvasSize({
width: canvasWidth,
height: canvasHeight
})
}

const imgElement = document.getElementById('image-stream');

// Temporarily remove the stream source
imgElement.src = '';

updateCanvasSize()

// Reapply the stream URL after the resize is complete
imgElement.src = streamUrl;

// Set resize listener
window.addEventListener('resize', updateCanvasSize);

return () => {
window.stop()
window.removeEventListener('resize', updateCanvasSize);
}
}, []);
}, [isPortrait]);

return (
<div
id='phone-imitation'
>
<h3
style={{
color: '#2f3b26',
display: 'flex',
fontFamily: 'Verdana',
justifyContent: 'center'
}}
>{deviceData.model}</h3>
<Grid>
<div
id="stream-div"
style={{
width: streamData.canvasWidth,
height: streamData.canvasHeight
}}
id='phone-imitation'
>
<Canvas
canvasWidth={streamData.canvasWidth}
canvasHeight={streamData.canvasHeight}
authToken={authToken}
logout={logout}
streamData={streamData}
setDialog={setDialog}
></Canvas>
<Stream
canvasWidth={streamData.canvasWidth}
canvasHeight={streamData.canvasHeight}
streamUrl={streamUrl}
></Stream>
</div>
<Divider></Divider>
<Grid
height='50px'
display='flex'
justifyContent='center'
style={{
marginTop: '10px'
}}
>
<Button
onClick={() => homeButton(authToken, deviceData, setDialog)}
className='canvas-buttons'
startIcon={<HomeIcon />}
variant='contained'
<h3
style={{
fontWeight: "bold",
color: "#9ba984",
backgroundColor: "#2f3b26",
borderBottomLeftRadius: '25px',
color: '#2f3b26',
display: 'flex',
fontFamily: 'Verdana',
justifyContent: 'center'
}}
>Home</Button>
<Button
onClick={() => lockButton(authToken, deviceData, setDialog)}
className='canvas-buttons'
startIcon={<LockIcon />}
variant='contained'
>{deviceData.model}</h3>
<div
id="stream-div"
style={{
fontWeight: "bold",
color: "#9ba984",
backgroundColor: "#2f3b26"
width: streamData.canvasWidth,
height: streamData.canvasHeight
}}
>Lock</Button>
<Button
onClick={() => unlockButton(authToken, deviceData, setDialog)}
className='canvas-buttons'
startIcon={<LockOpenIcon />}
variant='contained'
>
<Canvas
canvasWidth={streamData.canvasWidth}
canvasHeight={streamData.canvasHeight}
authToken={authToken}
logout={logout}
streamData={streamData}
setDialog={setDialog}
></Canvas>
<Stream
canvasWidth={streamData.canvasWidth}
canvasHeight={streamData.canvasHeight}
streamUrl={streamUrl}
></Stream>
</div>
<Divider></Divider>
<Grid
height='50px'
display='flex'
justifyContent='center'
style={{
fontWeight: "bold",
color: "#9ba984",
backgroundColor: "#2f3b26",
borderBottomRightRadius: '25px'
marginTop: '10px'
}}
>Unlock</Button>
</Grid>
</div >

>
<Button
onClick={() => homeButton(authToken, deviceData, setDialog)}
className='canvas-buttons'
startIcon={<HomeIcon />}
variant='contained'
style={{
fontWeight: "bold",
color: "#9ba984",
backgroundColor: "#2f3b26",
borderBottomLeftRadius: '25px',
}}
>Home</Button>
<Button
onClick={() => lockButton(authToken, deviceData, setDialog)}
className='canvas-buttons'
startIcon={<LockIcon />}
variant='contained'
style={{
fontWeight: "bold",
color: "#9ba984",
backgroundColor: "#2f3b26"
}}
>Lock</Button>
<Button
onClick={() => unlockButton(authToken, deviceData, setDialog)}
className='canvas-buttons'
startIcon={<LockOpenIcon />}
variant='contained'
style={{
fontWeight: "bold",
color: "#9ba984",
backgroundColor: "#2f3b26",
borderBottomRightRadius: '25px'
}}
>Unlock</Button>
</Grid>
</div >
<Tooltip
title="This does not change the orientation of the device itself, just updates the UI if the device orientation is already changed"
arrow
placement='bottom'
>
<Grid
display='flex'
justifyContent='center'
style={{
marginTop: '10px'
}}
>
<Button
variant={"contained"}
color={"secondary"}
onClick={() => handleOrientationButtonClick(true)}
disabled={isPortrait}
>
Portrait
</Button>
<Button
variant={"contained"}
color={"secondary"}
onClick={() => handleOrientationButtonClick(false)}
disabled={!isPortrait}
>
Landscape
</Button>
</Grid>
</Tooltip>
</Grid>
)
}

Expand All @@ -150,9 +203,28 @@ function Canvas({ authToken, logout, streamData, setDialog }) {

function getCursorCoordinates(event) {
const rect = event.currentTarget.getBoundingClientRect()
const x = event.clientX - rect.left
const y = event.clientY - rect.top
return [x, y];
// If its portrait use the usual calculation for tap coordinates
if (streamData.isPortrait) {
const x = event.clientX - rect.left
const y = event.clientY - rect.top
return [x, y]
}
// If its landscape and the provider uses custom wda for tapping/swiping
if (streamData.device_os === 'ios' && streamData.uses_custom_wda) {
// Have to subtract the tap coordinate from the canvas height
// Because the custom wda tap uses coordinates where in landscape
// x starts from the bottom(being essentially reversed y)
// and y starts from the left(being essentially x)
const x = streamData.canvasHeight - (event.clientY - rect.top)
const y = event.clientX - rect.left
return [x, y]
}
// If its landscape and provider does not use custom wda for tapping/swiping
// just reverse x and y
const x = event.clientY - rect.top
const y = event.clientX - rect.left
console.log("Returning " + x + " " + y)
return [y, x];
}

function handleMouseDown(event) {
Expand Down Expand Up @@ -218,8 +290,13 @@ function tapCoordinates(authToken, logout, pos, streamData, setDialog) {

// if the stream height
if (streamData.canvasHeight != streamData.deviceY) {
x = (x / streamData.canvasWidth) * streamData.deviceX
y = (y / streamData.canvasHeight) * streamData.deviceY
if (streamData.isPortrait) {
x = (x / streamData.canvasWidth) * streamData.deviceX
y = (y / streamData.canvasHeight) * streamData.deviceY
} else {
x = (x / streamData.canvasHeight) * streamData.deviceX
y = (y / streamData.canvasWidth) * streamData.deviceY
}
}

let jsonData = JSON.stringify({
Expand Down Expand Up @@ -287,10 +364,22 @@ function swipeCoordinates(authToken, logout, coord1, coord2, streamData, setDial

// if the stream height
if (streamData.canvasHeight != streamData.deviceY) {
firstCoordX = (firstCoordX / streamData.canvasWidth) * streamData.deviceX
firstCoordY = (firstCoordY / streamData.canvasHeight) * streamData.deviceY
secondCoordX = (secondCoordX / streamData.canvasWidth) * streamData.deviceX
secondCoordY = (secondCoordY / streamData.canvasHeight) * streamData.deviceY
// If the device is landscape and is android
// We need to switch the coordinate calculation
// Divide by height for X and divide by width for Y
if (streamData.device_os === 'android' && !streamData.isPortrait) {
firstCoordX = (firstCoordX / streamData.canvasHeight) * streamData.deviceX
firstCoordY = (firstCoordY / streamData.canvasWidth) * streamData.deviceY
secondCoordX = (secondCoordX / streamData.canvasHeight) * streamData.deviceX
secondCoordY = (secondCoordY / streamData.canvasWidth) * streamData.deviceY
} else {
// If the device is not in landscape and is not android
// Divide as usual - X by width and Y by height
firstCoordX = (firstCoordX / streamData.canvasWidth) * streamData.deviceX
firstCoordY = (firstCoordY / streamData.canvasHeight) * streamData.deviceY
secondCoordX = (secondCoordX / streamData.canvasWidth) * streamData.deviceX
secondCoordY = (secondCoordY / streamData.canvasHeight) * streamData.deviceY
}
}

let jsonData = JSON.stringify({
Expand Down
1 change: 1 addition & 0 deletions provider/router/routes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,6 +238,7 @@ func DeviceInfo(c *gin.Context) {

if dev, ok := devices.DBDeviceMap[udid]; ok {
devices.UpdateInstalledApps(dev)
dev.UsesCustomWDA = config.ProviderConfig.UseCustomWDA
c.JSON(http.StatusOK, dev)
return
}
Expand Down

0 comments on commit 21b9fcc

Please sign in to comment.