Merge pull request #116 from shamanec/add-device-reprovision-button

Add device re-provision button

common/models/models.go

@@ -63,6 +63,7 @@ type Device struct {
 	WDAStreamPort    string             `json:"-" bson:"-"` // port assigned to iOS devices for the WebDriverAgent stream
 	WDAPort          string             `json:"-" bson:"-"` // port assigned to iOS devices for the WebDriverAgent instance
 	WdaReadyChan     chan bool          `json:"-" bson:"-"` // channel for checking that WebDriverAgent is up after start
+	AppiumReadyChan  chan bool          `json:"-" bson:"-"` // channel for checking that Appium is up after start
 	Context          context.Context    `json:"-" bson:"-"` // context used to control the device set up since we have multiple goroutines
 	CtxCancel        context.CancelFunc `json:"-" bson:"-"` // cancel func for the context above, can be used to stop all running device goroutines
 	GoIOSDeviceEntry ios.DeviceEntry    `json:"-" bson:"-"` // `go-ios` device entry object used for `go-ios` library interactions

hub/gads-ui/src/components/Admin/Devices/DevicesAdministration.js

@@ -142,7 +142,7 @@ function NewDevice({ providers, handleGetDeviceData }) {
                 width: '400px',
                 minWidth: '400px',
                 maxWidth: '400px',
-                height: '780px',
+                height: '830px',
                 borderRadius: '5px',
                 backgroundColor: '#9ba984'
             }}
@@ -345,8 +345,10 @@ function ExistingDevice({ deviceData, providersData, handleGetDeviceData }) {
     const [type, setType] = useState(deviceData.device_type)
     const udid = deviceData.udid
 
-    const [loading, setLoading] = useState(false);
+    const [loading, setLoading] = useState(false)
+    const [reprovisionLoading, setReprovisionLoading] = useState(false)
     const [updateDeviceStatus, setUpdateDeviceStatus] = useState(null)
+    const [reprovisionDeviceStatus, setReprovisionDeviceStatus] = useState(null)
 
     useEffect(() => {
         setProvider(deviceData.provider)
@@ -394,6 +396,30 @@ function ExistingDevice({ deviceData, providersData, handleGetDeviceData }) {
             })
     }
 
+    function handleReprovisionDevice(event) {
+        setReprovisionLoading(true)
+        setReprovisionDeviceStatus(null)
+        event.preventDefault()
+
+        let url = `/device/${udid}/reset`
+
+        api.post(url)
+            .then(() => {
+                setReprovisionDeviceStatus('success')
+            })
+            .catch(() => {
+                setReprovisionDeviceStatus('error')
+            })
+            .finally(() => {
+                setTimeout(() => {
+                    setReprovisionLoading(false)
+                    setTimeout(() => {
+                        setReprovisionDeviceStatus(null)
+                    }, 2000)
+                }, 1000)
+            })
+    }
+
     function handleDeleteDevice(event) {
         event.preventDefault()
 
@@ -417,7 +443,7 @@ function ExistingDevice({ deviceData, providersData, handleGetDeviceData }) {
                 width: '400px',
                 minWidth: '400px',
                 maxWidth: '400px',
-                height: '780px',
+                height: '830px',
                 borderRadius: '5px',
                 backgroundColor: '#9ba984'
             }}
@@ -601,6 +627,28 @@ function ExistingDevice({ deviceData, providersData, handleGetDeviceData }) {
                             'Update device'
                         )}
                     </Button>
+                    <Button
+                        variant="contained"
+                        style={{
+                            backgroundColor: '#2f3b26',
+                            color: '#f4e6cd',
+                            fontWeight: "bold",
+                            boxShadow: 'none',
+                            height: '40px'
+                        }}
+                        onClick={handleReprovisionDevice}
+                        disabled={reprovisionLoading || reprovisionDeviceStatus === 'success' || reprovisionDeviceStatus === 'error'}
+                    >
+                        {reprovisionLoading ? (
+                            <CircularProgress size={25} style={{ color: '#f4e6cd' }} />
+                        ) : reprovisionDeviceStatus === 'success' ? (
+                            <CheckIcon size={25} style={{ color: '#f4e6cd', stroke: '#f4e6cd', strokeWidth: 2 }} />
+                        ) : reprovisionDeviceStatus === 'error' ? (
+                            <CloseIcon size={25} style={{ color: 'red', stroke: 'red', strokeWidth: 2 }} />
+                        ) : (
+                            'Re-provision device'
+                        )}
+                    </Button>
                     <Button
                         onClick={() => setOpenAlert(true)}
                         style={{

hub/router/appiumgrid.go

@@ -59,7 +59,12 @@ func UpdateExpiredGridSessions() {
 	for {
 		devices.HubDevicesData.Mu.Lock()
 		for _, hubDevice := range devices.HubDevicesData.Devices {
-			if hubDevice.LastAutomationActionTS <= (time.Now().UnixMilli()-hubDevice.AppiumNewCommandTimeout) && hubDevice.IsRunningAutomation {
+			// Reset device if its not connected
+			// Or it hasn't received any Appium requests in the command timeout and is running automation
+			// Or if its provider state is not "live" - device was re-provisioned for example
+			if !hubDevice.Device.Connected ||
+				(hubDevice.LastAutomationActionTS <= (time.Now().UnixMilli()-hubDevice.AppiumNewCommandTimeout) && hubDevice.IsRunningAutomation) ||
+				hubDevice.Device.ProviderState != "live" {
 				hubDevice.IsRunningAutomation = false
 				hubDevice.IsAvailableForAutomation = true
 				hubDevice.SessionID = ""
@@ -69,7 +74,7 @@ func UpdateExpiredGridSessions() {
 			}
 		}
 		devices.HubDevicesData.Mu.Unlock()
-		time.Sleep(3 * time.Second)
+		time.Sleep(1 * time.Second)
 	}
 }
 
@@ -182,6 +187,23 @@ func AppiumGridMiddleware() gin.HandlerFunc {
 			}
 			defer resp.Body.Close()
 
+			if resp.StatusCode == http.StatusInternalServerError {
+				// Start a goroutine that will release the device after 5 seconds if no other actions were taken
+				go func() {
+					time.Sleep(10 * time.Second)
+					devices.HubDevicesData.Mu.Lock()
+					if foundDevice.LastAutomationActionTS <= (time.Now().UnixMilli() - 5000) {
+						foundDevice.IsAvailableForAutomation = true
+						foundDevice.SessionID = ""
+						foundDevice.IsRunningAutomation = false
+						foundDevice.InUseBy = ""
+					}
+					devices.HubDevicesData.Mu.Unlock()
+				}()
+				c.JSON(http.StatusInternalServerError, createErrorResponse("GADS got an internal server error from the proxy session request to the device respective provider Appium endpoint", "", ""))
+				return
+			}
+
 			// Read the response sessionRequestBody from the proxied request
 			proxiedSessionResponseBody, err := readBody(resp.Body)
 			if err != nil {
@@ -201,7 +223,7 @@ func AppiumGridMiddleware() gin.HandlerFunc {
 				foundDevice.IsAvailableForAutomation = true
 				foundDevice.IsRunningAutomation = false
 				devices.HubDevicesData.Mu.Unlock()
-				c.JSON(http.StatusInternalServerError, createErrorResponse("GADS failed to unmarshal the response sessionRequestBody of the proxied Appium session request", "", err.Error()))
+				c.JSON(http.StatusInternalServerError, createErrorResponse("GADS failed to unmarshal the response sessionRequestBody of the proxied Appium session request "+err.Error(), "", err.Error()))
 				return
 			}
 
@@ -276,7 +298,14 @@ func AppiumGridMiddleware() gin.HandlerFunc {
 			}()
 
 			// Create a new request to the device target URL on its provider instance
-			proxyReq, err := http.NewRequest(c.Request.Method, fmt.Sprintf("http://%s/device/%s/appium%s", foundDevice.Device.Host, foundDevice.Device.UDID, strings.Replace(c.Request.URL.Path, "/grid", "", -1)), bytes.NewBuffer(origRequestBody))
+			proxyReq, err := http.NewRequest(
+				c.Request.Method,
+				fmt.Sprintf("http://%s/device/%s/appium%s",
+					foundDevice.Device.Host,
+					foundDevice.Device.UDID,
+					strings.Replace(c.Request.URL.Path, "/grid", "", -1)),
+				bytes.NewBuffer(origRequestBody),
+			)
 			if err != nil {
 				c.JSON(http.StatusInternalServerError, createErrorResponse("GADS failed to create proxy request for this call", "", err.Error()))
 				return
@@ -314,6 +343,23 @@ func AppiumGridMiddleware() gin.HandlerFunc {
 				}()
 			}
 
+			if resp.StatusCode == http.StatusInternalServerError {
+				// Start a goroutine that will release the device after 10 seconds if no other actions were taken
+				go func() {
+					time.Sleep(10 * time.Second)
+					devices.HubDevicesData.Mu.Lock()
+					if foundDevice.LastAutomationActionTS <= (time.Now().UnixMilli() - 10000) {
+						foundDevice.SessionID = ""
+						foundDevice.IsAvailableForAutomation = true
+						foundDevice.IsRunningAutomation = false
+						foundDevice.InUseBy = ""
+					}
+					devices.HubDevicesData.Mu.Unlock()
+				}()
+				c.JSON(http.StatusInternalServerError, createErrorResponse("GADS got an internal server error from the proxy request to the device respective provider Appium endpoint", "", ""))
+				return
+			}
+
 			// Read the response origRequestBody of the proxied request
 			proxiedRequestBody, err := readBody(resp.Body)
 			if err != nil {
@@ -390,6 +436,8 @@ func findAvailableDevice(caps CommonCapabilities) (*models.LocalHubDevice, error
 			for _, localDevice := range devices.HubDevicesData.Devices {
 				if strings.EqualFold(localDevice.Device.OS, "ios") &&
 					!localDevice.InUse &&
+					localDevice.Device.Connected &&
+					localDevice.Device.ProviderState == "live" &&
 					localDevice.Device.LastUpdatedTimestamp >= (time.Now().UnixMilli()-3000) &&
 					localDevice.IsAvailableForAutomation &&
 					localDevice.Device.Usage != "control" &&
@@ -405,6 +453,8 @@ func findAvailableDevice(caps CommonCapabilities) (*models.LocalHubDevice, error
 			for _, localDevice := range devices.HubDevicesData.Devices {
 				if strings.EqualFold(localDevice.Device.OS, "android") &&
 					!localDevice.InUse &&
+					localDevice.Device.Connected &&
+					localDevice.Device.ProviderState == "live" &&
 					localDevice.Device.LastUpdatedTimestamp >= (time.Now().UnixMilli()-3000) &&
 					localDevice.IsAvailableForAutomation &&
 					localDevice.Device.Usage != "control" &&

hub/router/routes.go

@@ -706,6 +706,16 @@ func ProviderUpdate(c *gin.Context) {
 		devices.HubDevicesData.Mu.Lock()
 		hubDevice, ok := devices.HubDevicesData.Devices[providerDevice.UDID]
 		if ok {
+			// If device is not connected reset all fields that might allow it to get stuck in Running automation state
+			// If its not connected, then its not running automation or is available for automation
+			if !providerDevice.Connected {
+				hubDevice.IsAvailableForAutomation = false
+				hubDevice.IsRunningAutomation = false
+				hubDevice.InUseBy = ""
+				hubDevice.SessionID = ""
+				devices.HubDevicesData.Mu.Unlock()
+				continue
+			}
 			// Set a timestamp to indicate last time info about the device was updated from the provider
 			providerDevice.LastUpdatedTimestamp = time.Now().UnixMilli()
 

provider/devices/common.go

@@ -161,7 +161,7 @@ func setupDevices() {
 }
 
 func updateDevices() {
-	ticker := time.NewTicker(5 * time.Second)
+	ticker := time.NewTicker(1 * time.Second)
 	defer ticker.Stop()
 
 	for range ticker.C {
@@ -178,6 +178,7 @@ func updateDevices() {
 					setContext(dbDevice)
 					if dbDevice.OS == "ios" {
 						dbDevice.WdaReadyChan = make(chan bool, 1)
+						dbDevice.AppiumReadyChan = make(chan bool, 1)
 						go setupIOSDevice(dbDevice)
 					}
 
@@ -227,6 +228,14 @@ func setupAndroidDevice(device *models.Device) {
 	}
 	device.StreamPort = streamPort
 
+	appiumPort, err := providerutil.GetFreePort()
+	if err != nil {
+		logger.ProviderLogger.LogError("android_device_setup", fmt.Sprintf("Could not allocate free host port for Appium for device `%v` - %v", device.UDID, err))
+		resetLocalDevice(device)
+		return
+	}
+	device.AppiumPort = appiumPort
+
 	apps := GetInstalledAppsAndroid(device)
 	if slices.Contains(apps, "com.shamanec.stream") {
 		stopGadsStreamService(device)
@@ -300,6 +309,18 @@ func setupAndroidDevice(device *models.Device) {
 	}
 
 	go startAppium(device)
+	go checkAppiumUp(device)
+
+	select {
+	case <-device.AppiumReadyChan:
+		logger.ProviderLogger.LogInfo("ios_device_setup", fmt.Sprintf("Successfully started Appium for device `%v` on port %v", device.UDID, device.AppiumPort))
+		break
+	case <-time.After(60 * time.Second):
+		logger.ProviderLogger.LogError("ios_device_setup", fmt.Sprintf("Did not successfully start Appium for device `%v` in 60 seconds", device.UDID))
+		resetLocalDevice(device)
+		return
+	}
+
 	if config.ProviderConfig.UseSeleniumGrid {
 		go startGridNode(device)
 	}
@@ -386,6 +407,14 @@ func setupIOSDevice(device *models.Device) {
 	}
 	device.WDAStreamPort = wdaStreamPort
 
+	appiumPort, err := providerutil.GetFreePort()
+	if err != nil {
+		logger.ProviderLogger.LogError("ios_device_setup", fmt.Sprintf("Could not allocate free Appium port for device `%v` - %v", device.UDID, err))
+		resetLocalDevice(device)
+		return
+	}
+	device.AppiumPort = appiumPort
+
 	// Forward the WebDriverAgent server and stream to the host
 	go goIOSForward(device, device.WDAPort, "8100")
 	go goIOSForward(device, device.StreamPort, "9500")
@@ -425,7 +454,7 @@ func setupIOSDevice(device *models.Device) {
 		logger.ProviderLogger.LogInfo("ios_device_setup", fmt.Sprintf("Successfully started WebDriverAgent for device `%v` forwarded on port %v", device.UDID, device.WDAPort))
 		break
 	case <-time.After(60 * time.Second):
-		logger.ProviderLogger.LogError("ios_device_setup", fmt.Sprintf("Did not successfully start WebDriverAgent on device `%v` in 30 seconds", device.UDID))
+		logger.ProviderLogger.LogError("ios_device_setup", fmt.Sprintf("Did not successfully start WebDriverAgent on device `%v` in 60 seconds", device.UDID))
 		resetLocalDevice(device)
 		return
 	}
@@ -439,6 +468,19 @@ func setupIOSDevice(device *models.Device) {
 	}
 
 	go startAppium(device)
+	go checkAppiumUp(device)
+
+	// Wait until WebDriverAgent successfully starts
+	select {
+	case <-device.AppiumReadyChan:
+		logger.ProviderLogger.LogInfo("ios_device_setup", fmt.Sprintf("Successfully started Appium for device `%v` on port %v", device.UDID, device.AppiumPort))
+		break
+	case <-time.After(60 * time.Second):
+		logger.ProviderLogger.LogError("ios_device_setup", fmt.Sprintf("Did not successfully start Appium for device `%v` in 60 seconds", device.UDID))
+		resetLocalDevice(device)
+		return
+	}
+
 	if config.ProviderConfig.UseSeleniumGrid {
 		go startGridNode(device)
 	}
@@ -577,16 +619,6 @@ func startAppium(device *models.Device) {
 	}
 
 	capabilitiesJson, _ := json.Marshal(capabilities)
-
-	// Get a free port on the host for Appium server
-	appiumPort, err := providerutil.GetFreePort()
-	if err != nil {
-		logger.ProviderLogger.LogError("device_setup", fmt.Sprintf("startAppium: Could not allocate free Appium host port for device - %v, err - %v", device.UDID, err))
-		resetLocalDevice(device)
-		return
-	}
-	device.AppiumPort = appiumPort
-
 	cmd := exec.CommandContext(
 		device.Context,
 		"appium",

provider/devices/ios.go

@@ -17,6 +17,7 @@ import (
 	"GADS/common/models"
 	"GADS/provider/config"
 	"GADS/provider/logger"
+
 	"github.com/Masterminds/semver"
 	"github.com/danielpaulus/go-ios/ios"
 )
@@ -445,3 +446,28 @@ func checkWebDriverAgentUp(device *models.Device) {
 		loops++
 	}
 }
+
+func checkAppiumUp(device *models.Device) {
+	var netClient = &http.Client{
+		Timeout: time.Second * 120,
+	}
+
+	req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://localhost:%v/status", device.AppiumPort), nil)
+
+	loops := 0
+	for {
+		if loops >= 30 {
+			return
+		}
+		resp, err := netClient.Do(req)
+		if err != nil {
+			time.Sleep(1 * time.Second)
+		} else {
+			if resp.StatusCode == http.StatusOK {
+				device.AppiumReadyChan <- true
+				return
+			}
+		}
+		loops++
+	}
+}

provider/router/routes.go

@@ -319,6 +319,14 @@ func ResetDevice(c *gin.Context) {
 	udid := c.Param("udid")
 
 	if device, ok := devices.DBDeviceMap[udid]; ok {
+		if device.IsResetting {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Device setup is already being reset"})
+			return
+		}
+		if device.ProviderState != "live" {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": "Only devices in `live` state can be reset, current state is `" + device.ProviderState + "`"})
+			return
+		}
 		device.IsResetting = true
 		device.CtxCancel()
 		device.ProviderState = "init"