Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feat[TE-20444]: Added required nlps in mobile_web for image_based_actions addon #60

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Feat[TE-20444]: Added required nlps in mobile_web for image_based_actions addon #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion image_based_actions/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.testsigma.addons</groupId>
<artifactId>image_based_actions</artifactId>
<version>1.0.4</version>
<version>1.0.8</version>
<packaging>jar</packaging>

<properties>
Expand Down
111 changes: 111 additions & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
package com.testsigma.addons.ios;

import com.testsigma.sdk.ApplicationType;
import com.testsigma.sdk.FindImageResponse;
import com.testsigma.sdk.IOSAction;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.OCR;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
import io.appium.java_client.ios.IOSDriver;
import lombok.Data;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.interactions.Pause;
import org.openqa.selenium.interactions.PointerInput;
import org.openqa.selenium.interactions.Sequence;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.util.Arrays;

import static java.time.Duration.ofMillis;
import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH;
import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT;
import static org.openqa.selenium.interactions.PointerInput.Origin.viewport;

@Data
@Action(actionText = "Tap on image image-url",
description = "Tap on give image",
applicationType = ApplicationType.IOS)
public class ClickOnImage extends IOSAction {
@TestData(reference = "image-url")
private com.testsigma.sdk.TestData testData1;

@OCR
private com.testsigma.sdk.OCR ocr;

@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;

@Override
protected Result execute() throws NoSuchElementException {
Result result = Result.SUCCESS;
try {
IOSDriver iosDriver = (IOSDriver) this.driver;
File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE);
BufferedImage bufferedImage = ImageIO.read(baseImageFile);
int imageWidth = bufferedImage.getWidth();
int imageHeight = bufferedImage.getHeight();
logger.info("Width of image: " + imageWidth);
logger.info("Height of image: " + imageHeight);
Dimension dimension = iosDriver.manage().window().getSize();
int screenWidth = dimension.width;
int screenHeight = dimension.height;
logger.info("Screen width: "+screenWidth);
logger.info("Screen height: "+screenHeight);
String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
FindImageResponse response = ocr.findImage(testData1.getValue().toString());
if(response.getIsFound()) {
logger.info("Image location found");
logger.info("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
logger.info("Performing click..");
int x = (response.getX1() + response.getX2()) / 2;
int y = (response.getY1() + response.getY2()) / 2;
logger.info("Screen shot based click locations: x="+x+"y="+y);
double xRelative = ((double) x / imageWidth);
double yRelative = ((double) y / imageHeight);
logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative);
int clickLocationX;
int clickLocationY;
if (Math.abs(imageWidth-screenWidth) > 20) {
clickLocationX = (int) (xRelative * screenWidth);
clickLocationY = (int) (yRelative * screenHeight);
} else {
clickLocationX = x;
clickLocationY = y;
}
logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY);
PointerInput FINGER = new PointerInput(TOUCH, "finger");
Sequence tap = new Sequence(FINGER, 1)
.addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY))
.addAction(FINGER.createPointerDown(LEFT.asArg()))
.addAction(new Pause(FINGER, ofMillis(2)))
.addAction(FINGER.createPointerUp(LEFT.asArg()));
iosDriver.perform(Arrays.asList(tap));
logger.info("CLick performed");
setSuccessMessage("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
Thread.sleep(1000);
} else {
setErrorMessage("Unable to fetch the coordinates");
result = Result.FAILED;
}

}
catch (Exception e){
logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click action");
result = Result.FAILED;
}
return result;
}
}
113 changes: 113 additions & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageOccurrenceBased.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package com.testsigma.addons.ios;

import com.testsigma.sdk.ApplicationType;
import com.testsigma.sdk.FindImageResponse;
import com.testsigma.sdk.IOSAction;
import com.testsigma.sdk.Result;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.OCR;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
import io.appium.java_client.ios.IOSDriver;
import lombok.Data;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.interactions.Pause;
import org.openqa.selenium.interactions.PointerInput;
import org.openqa.selenium.interactions.Sequence;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.util.Arrays;

import static java.time.Duration.ofMillis;
import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH;
import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT;
import static org.openqa.selenium.interactions.PointerInput.Origin.viewport;

@Data
@Action(actionText = "Tap on image image-url, occurrence position found-at-position",
description = "Tap on give image at the given position",
applicationType = ApplicationType.IOS)
public class ClickOnImageOccurrenceBased extends IOSAction {
@TestData(reference = "image-url")
private com.testsigma.sdk.TestData testData1;

@TestData(reference = "found-at-position")
private com.testsigma.sdk.TestData testData2;

@OCR
private com.testsigma.sdk.OCR ocr;

@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;

@Override
protected Result execute() throws NoSuchElementException {
Result result = Result.SUCCESS;
try {
IOSDriver iosDriver = (IOSDriver) this.driver;
File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE);
BufferedImage bufferedImage = ImageIO.read(baseImageFile);
int imageWidth = bufferedImage.getWidth();
int imageHeight = bufferedImage.getHeight();
logger.info("Width of image: " + imageWidth);
logger.info("Height of image: " + imageHeight);
Dimension dimension = iosDriver.manage().window().getSize();
int screenWidth = dimension.width;
int screenHeight = dimension.height;
logger.info("Screen width: "+screenWidth);
logger.info("Screen height: "+screenHeight);
String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Integer.parseInt(testData2.getValue().toString()));
if(response.getIsFound()) {
logger.info("Image location found");
logger.info("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
logger.info("Performing click..");
int x = (response.getX1() + response.getX2()) / 2;
int y = (response.getY1() + response.getY2()) / 2;
logger.info("Screen shot based click locations: x="+x+"y="+y);
double xRelative = ((double) x / imageWidth);
double yRelative = ((double) y / imageHeight);
logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative);
int clickLocationX;
int clickLocationY;
if(Math.abs(imageWidth-screenWidth) > 20){
clickLocationX = (int) (xRelative * screenWidth);
clickLocationY = (int) (yRelative * screenHeight);
} else {
clickLocationX = x;
clickLocationY = y;
}
logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY);
PointerInput FINGER = new PointerInput(TOUCH, "finger");
Sequence tap = new Sequence(FINGER, 1)
.addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY))
.addAction(FINGER.createPointerDown(LEFT.asArg()))
.addAction(new Pause(FINGER, ofMillis(2)))
.addAction(FINGER.createPointerUp(LEFT.asArg()));
iosDriver.perform(Arrays.asList(tap));
logger.info("CLick performed");
setSuccessMessage("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
Thread.sleep(1000);
} else {
setErrorMessage("Unable to fetch the coordinates");
result = Result.FAILED;
}
}
catch (Exception e){
logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click action");
result = Result.FAILED;
}
return result;
}
}
110 changes: 110 additions & 0 deletions image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThreshold.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package com.testsigma.addons.ios;

import com.testsigma.sdk.*;
import com.testsigma.sdk.annotation.Action;
import com.testsigma.sdk.annotation.OCR;
import com.testsigma.sdk.annotation.TestData;
import com.testsigma.sdk.annotation.TestStepResult;
import io.appium.java_client.ios.IOSDriver;
import lombok.Data;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.openqa.selenium.Dimension;
import org.openqa.selenium.NoSuchElementException;
import org.openqa.selenium.OutputType;
import org.openqa.selenium.TakesScreenshot;
import org.openqa.selenium.interactions.Pause;
import org.openqa.selenium.interactions.PointerInput;
import org.openqa.selenium.interactions.Sequence;

import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.File;
import java.util.Arrays;

import static java.time.Duration.ofMillis;
import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH;
import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT;
import static org.openqa.selenium.interactions.PointerInput.Origin.viewport;

@Data
@Action(actionText = "Tap on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match)",
description = "Tap on given image with threshold",
applicationType = ApplicationType.IOS)
public class ClickOnImageWithThreshold extends IOSAction {
@TestData(reference = "image-file")
private com.testsigma.sdk.TestData testData1;
@TestData(reference = "threshold-value")
private com.testsigma.sdk.TestData testData2;


@OCR
private com.testsigma.sdk.OCR ocr;

@TestStepResult
private com.testsigma.sdk.TestStepResult testStepResult;

@Override
protected Result execute() throws NoSuchElementException {
Result result = Result.SUCCESS;
try {
IOSDriver iosDriver = (IOSDriver) this.driver;
File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE);
BufferedImage bufferedImage = ImageIO.read(baseImageFile);
int imageWidth = bufferedImage.getWidth();
int imageHeight = bufferedImage.getHeight();
logger.info("Width of image: " + imageWidth);
logger.info("Height of image: " + imageHeight);
Dimension dimension = iosDriver.manage().window().getSize();
int screenWidth = dimension.width;
int screenHeight = dimension.height;
logger.info("Screen width: "+screenWidth);
logger.info("Screen height: "+screenHeight);
String url = testStepResult.getScreenshotUrl();
logger.info("Amazon s3 url in which we are storing base image"+url);
ocr.uploadFile(url, baseImageFile);
FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString()));
if(response.getIsFound()) {
logger.info("Image location found");
logger.info("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
logger.info("Performing click..");
int x = (response.getX1() + response.getX2()) / 2;
int y = (response.getY1() + response.getY2()) / 2;
logger.info("Screen shot based click locations: x="+x+"y="+y);
double xRelative = ((double) x / imageWidth);
double yRelative = ((double) y / imageHeight);
logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative);
int clickLocationX;
int clickLocationY;
if(Math.abs(imageWidth-screenWidth) > 20){
clickLocationX = (int) (xRelative * screenWidth);
clickLocationY = (int) (yRelative * screenHeight);
} else {
clickLocationX = x;
clickLocationY = y;
}
logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY);
PointerInput FINGER = new PointerInput(TOUCH, "finger");
Sequence tap = new Sequence(FINGER, 1)
.addAction(FINGER.createPointerMove(ofMillis(0), viewport(), x, y))
.addAction(FINGER.createPointerDown(LEFT.asArg()))
.addAction(new Pause(FINGER, ofMillis(2)))
.addAction(FINGER.createPointerUp(LEFT.asArg()));
iosDriver.perform(Arrays.asList(tap));
logger.info("CLick performed");
setSuccessMessage("Image Found :" + response.getIsFound() +
" Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2());
Thread.sleep(1000);
} else {
setErrorMessage("Unable to fetch the coordinates");
result = Result.FAILED;
}
}
catch (Exception e){
logger.info("Exception: "+ ExceptionUtils.getStackTrace(e));
setErrorMessage("Exception occurred while performing click action");
result = Result.FAILED;
}
return result;
}
}
Loading