From 6bcd65174b795353d196c1e16db7f7b6a00d871e Mon Sep 17 00:00:00 2001 From: Sivasai Podugu Date: Wed, 24 Jul 2024 14:09:54 +0530 Subject: [PATCH] Feat[TE-20444]: Added required nlps in mobile_web upgraded addon to 1.0.8 --- image_based_actions/pom.xml | 2 +- .../testsigma/addons/ios/ClickOnImage.java | 111 ++++++++++++ .../ios/ClickOnImageOccurrenceBased.java | 113 +++++++++++++ .../addons/ios/ClickOnImageWithThreshold.java | 110 ++++++++++++ ...ckOnImageWithThresholdOccurrenceBased.java | 115 +++++++++++++ .../com/testsigma/addons/ios/ClickOnText.java | 125 ++++++++++++++ .../ios/ClickOnTextOccurrenceBased.java | 134 +++++++++++++++ .../com/testsigma/addons/ios/SearchImage.java | 54 ++++++ .../addons/ios/SearchImageWithScale.java | 57 +++++++ .../addons/ios/SearchImageWithThreshold.java | 55 ++++++ .../addons/mobile_web/ClickOnImage.java | 65 +++++++ .../mobile_web/ClickOnImageWithThreshold.java | 71 ++++++++ .../addons/mobile_web/SearchImage.java | 34 ++++ .../mobile_web/SearchImageWithScale.java | 40 +++++ .../mobile_web/SearchImageWithThreshold.java | 37 ++++ .../mobile_web/util/ClickActionUtils.java | 109 ++++++++++++ .../addons/mobile_web/util/ContextUtils.java | 37 ++++ .../testsigma/addons/web/ClickOnImage.java | 2 +- .../addons/windows/DoubleClickOnImage.java | 101 +++++++++++ .../DoubleClickOnImageOccurrenceBased.java | 102 +++++++++++ .../DoubleClickOnImageWithThreshold.java | 101 +++++++++++ ...ckOnImageWithThresholdOccurrenceBased.java | 106 ++++++++++++ .../addons/windows/DoubleClickOnText.java | 151 +++++++++++++++++ .../DoubleClickOnTextOccurrenceBased.java | 160 ++++++++++++++++++ 24 files changed, 1990 insertions(+), 2 deletions(-) create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImage.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageOccurrenceBased.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThreshold.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThresholdOccurrenceBased.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnText.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnTextOccurrenceBased.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImage.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithScale.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithThreshold.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImage.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithScale.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithThreshold.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ClickActionUtils.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImage.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageOccurrenceBased.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThreshold.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThresholdOccurrenceBased.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnText.java create mode 100644 image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnTextOccurrenceBased.java diff --git a/image_based_actions/pom.xml b/image_based_actions/pom.xml index ec7648d1..6a7b70bc 100644 --- a/image_based_actions/pom.xml +++ b/image_based_actions/pom.xml @@ -6,7 +6,7 @@ 4.0.0 com.testsigma.addons image_based_actions - 1.0.4 + 1.0.8 jar diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImage.java new file mode 100644 index 00000000..ad8c7529 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImage.java @@ -0,0 +1,111 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.IOSAction; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +@Data +@Action(actionText = "Tap on image image-url", + description = "Tap on give image", + applicationType = ApplicationType.IOS) +public class ClickOnImage extends IOSAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + IOSDriver iosDriver = (IOSDriver) this.driver; + File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + BufferedImage bufferedImage = ImageIO.read(baseImageFile); + int imageWidth = bufferedImage.getWidth(); + int imageHeight = bufferedImage.getHeight(); + logger.info("Width of image: " + imageWidth); + logger.info("Height of image: " + imageHeight); + Dimension dimension = iosDriver.manage().window().getSize(); + int screenWidth = dimension.width; + int screenHeight = dimension.height; + logger.info("Screen width: "+screenWidth); + logger.info("Screen height: "+screenHeight); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage(testData1.getValue().toString()); + if(response.getIsFound()) { + logger.info("Image location found"); + logger.info("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + logger.info("Performing click.."); + int x = (response.getX1() + response.getX2()) / 2; + int y = (response.getY1() + response.getY2()) / 2; + logger.info("Screen shot based click locations: x="+x+"y="+y); + double xRelative = ((double) x / imageWidth); + double yRelative = ((double) y / imageHeight); + logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative); + int clickLocationX; + int clickLocationY; + if (Math.abs(imageWidth-screenWidth) > 20) { + clickLocationX = (int) (xRelative * screenWidth); + clickLocationY = (int) (yRelative * screenHeight); + } else { + clickLocationX = x; + clickLocationY = y; + } + logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY); + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + iosDriver.perform(Arrays.asList(tap)); + logger.info("CLick performed"); + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + Thread.sleep(1000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageOccurrenceBased.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageOccurrenceBased.java new file mode 100644 index 00000000..1140192a --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageOccurrenceBased.java @@ -0,0 +1,113 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.IOSAction; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +@Data +@Action(actionText = "Tap on image image-url, occurrence position found-at-position", + description = "Tap on give image at the given position", + applicationType = ApplicationType.IOS) +public class ClickOnImageOccurrenceBased extends IOSAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + + @TestData(reference = "found-at-position") + private com.testsigma.sdk.TestData testData2; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + IOSDriver iosDriver = (IOSDriver) this.driver; + File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + BufferedImage bufferedImage = ImageIO.read(baseImageFile); + int imageWidth = bufferedImage.getWidth(); + int imageHeight = bufferedImage.getHeight(); + logger.info("Width of image: " + imageWidth); + logger.info("Height of image: " + imageHeight); + Dimension dimension = iosDriver.manage().window().getSize(); + int screenWidth = dimension.width; + int screenHeight = dimension.height; + logger.info("Screen width: "+screenWidth); + logger.info("Screen height: "+screenHeight); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Integer.parseInt(testData2.getValue().toString())); + if(response.getIsFound()) { + logger.info("Image location found"); + logger.info("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + logger.info("Performing click.."); + int x = (response.getX1() + response.getX2()) / 2; + int y = (response.getY1() + response.getY2()) / 2; + logger.info("Screen shot based click locations: x="+x+"y="+y); + double xRelative = ((double) x / imageWidth); + double yRelative = ((double) y / imageHeight); + logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative); + int clickLocationX; + int clickLocationY; + if(Math.abs(imageWidth-screenWidth) > 20){ + clickLocationX = (int) (xRelative * screenWidth); + clickLocationY = (int) (yRelative * screenHeight); + } else { + clickLocationX = x; + clickLocationY = y; + } + logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY); + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + iosDriver.perform(Arrays.asList(tap)); + logger.info("CLick performed"); + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + Thread.sleep(1000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThreshold.java new file mode 100644 index 00000000..a5ddea6c --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThreshold.java @@ -0,0 +1,110 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +@Data +@Action(actionText = "Tap on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match)", + description = "Tap on given image with threshold", + applicationType = ApplicationType.IOS) +public class ClickOnImageWithThreshold extends IOSAction { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + IOSDriver iosDriver = (IOSDriver) this.driver; + File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + BufferedImage bufferedImage = ImageIO.read(baseImageFile); + int imageWidth = bufferedImage.getWidth(); + int imageHeight = bufferedImage.getHeight(); + logger.info("Width of image: " + imageWidth); + logger.info("Height of image: " + imageHeight); + Dimension dimension = iosDriver.manage().window().getSize(); + int screenWidth = dimension.width; + int screenHeight = dimension.height; + logger.info("Screen width: "+screenWidth); + logger.info("Screen height: "+screenHeight); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString())); + if(response.getIsFound()) { + logger.info("Image location found"); + logger.info("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + logger.info("Performing click.."); + int x = (response.getX1() + response.getX2()) / 2; + int y = (response.getY1() + response.getY2()) / 2; + logger.info("Screen shot based click locations: x="+x+"y="+y); + double xRelative = ((double) x / imageWidth); + double yRelative = ((double) y / imageHeight); + logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative); + int clickLocationX; + int clickLocationY; + if(Math.abs(imageWidth-screenWidth) > 20){ + clickLocationX = (int) (xRelative * screenWidth); + clickLocationY = (int) (yRelative * screenHeight); + } else { + clickLocationX = x; + clickLocationY = y; + } + logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY); + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), x, y)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + iosDriver.perform(Arrays.asList(tap)); + logger.info("CLick performed"); + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + Thread.sleep(1000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThresholdOccurrenceBased.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThresholdOccurrenceBased.java new file mode 100644 index 00000000..74d55f4f --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnImageWithThresholdOccurrenceBased.java @@ -0,0 +1,115 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +@Data +@Action(actionText = "Tap on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match), occurrence position found-at-position", + description = "Tap on given image with threshold at the given position", + applicationType = ApplicationType.IOS) +public class ClickOnImageWithThresholdOccurrenceBased extends IOSAction { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + @TestData(reference = "found-at-position") + private com.testsigma.sdk.TestData testData3; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + IOSDriver iosDriver = (IOSDriver) this.driver; + File baseImageFile = ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + BufferedImage bufferedImage = ImageIO.read(baseImageFile); + int imageWidth = bufferedImage.getWidth(); + int imageHeight = bufferedImage.getHeight(); + logger.info("Width of image: " + imageWidth); + logger.info("Height of image: " + imageHeight); + Dimension dimension = iosDriver.manage().window().getSize(); + int screenWidth = dimension.width; + int screenHeight = dimension.height; + logger.info("Screen width: "+screenWidth); + logger.info("Screen height: "+screenHeight); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage( + testData1.getValue().toString(), + Integer.parseInt(testData3.getValue().toString()), + Float.valueOf(testData2.getValue().toString()) + ); + if(response.getIsFound()) { + logger.info("Image location found"); + logger.info("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + logger.info("Performing click.."); + int x = (response.getX1() + response.getX2()) / 2; + int y = (response.getY1() + response.getY2()) / 2; + logger.info("Screen shot based click locations: x="+x+"y="+y); + double xRelative = ((double) x / imageWidth); + double yRelative = ((double) y / imageHeight); + logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative); + int clickLocationX; + int clickLocationY; + if(Math.abs(imageWidth-screenWidth) > 20){ + clickLocationX = (int) (xRelative * screenWidth); + clickLocationY = (int) (yRelative * screenHeight); + } else { + clickLocationX = x; + clickLocationY = y; + } + logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY); + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + iosDriver.perform(Arrays.asList(tap)); + logger.info("CLick performed"); + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + Thread.sleep(1000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnText.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnText.java new file mode 100644 index 00000000..e75f0bbb --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnText.java @@ -0,0 +1,125 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +@Data +@Action(actionText = "Tap on text name", + description = "Tap on the text using the text coordinates", + applicationType = ApplicationType.IOS) + +public class ClickOnText extends IOSAction { + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestData(reference = "name") + private com.testsigma.sdk.TestData text; + + @Override + protected Result execute() { + Result result = Result.SUCCESS; + try { + IOSDriver iosDriver = (IOSDriver) this.driver; + File baseImageFile = ((TakesScreenshot) iosDriver).getScreenshotAs(OutputType.FILE); + BufferedImage bufferedImage = ImageIO.read(baseImageFile); + int imageWidth = bufferedImage.getWidth(); + int imageHeight = bufferedImage.getHeight(); + logger.info("Width of image: " + imageWidth); + logger.info("Height of image: " + imageHeight); + Dimension dimension = iosDriver.manage().window().getSize(); + int screenWidth = dimension.width; + int screenHeight = dimension.height; + logger.info("Screen width: " + screenWidth); + logger.info("Screen height: " + screenHeight); + List textPoints = ocr.extractTextFromPage(); + printAllCoordinates(textPoints); + OCRTextPoint textPoint = getTextPointFromText(textPoints); + if (textPoint == null) { + result = Result.FAILED; + setErrorMessage("Given text is not found"); + + } else { + logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + + ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2()); + logger.info("Performing click.."); + int x = (textPoint.getX1() + textPoint.getX2()) / 2; + int y = (textPoint.getY1() + textPoint.getY2()) / 2; + logger.info("Screen shot based click locations: x="+x+"y="+y); + double xRelative = ((double) x / imageWidth); + double yRelative = ((double) y / imageHeight); + logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative); + int clickLocationX; + int clickLocationY; + if(Math.abs(imageWidth-screenWidth) > 20){ + clickLocationX = (int) (xRelative * screenWidth); + clickLocationY = (int) (yRelative * screenHeight); + } else { + clickLocationX = x; + clickLocationY = y; + } + logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY); + clickOnCoordinates(clickLocationX, clickLocationY); + setSuccessMessage("Click operation performed on the text " + + " Text coordinates :" + "x1-" + textPoint.getX1() + ", x2-" + textPoint.getX2() + ", y1-" + textPoint.getY1() + ", y2-" + textPoint.getY2()); + } + } catch(Exception e) { + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } + private OCRTextPoint getTextPointFromText(List textPoints) { + if(textPoints == null) { + return null; + } + for(OCRTextPoint textPoint: textPoints) { + if(text.getValue().equals(textPoint.getText())) { + return textPoint; + + } + } + return null; + } + + private void printAllCoordinates(List textPoints) { + for(OCRTextPoint textPoint: textPoints) { + logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n"); + } + } + public void clickOnCoordinates(int clickX, int clickY) { + + IOSDriver iosDriver = (IOSDriver) this.driver; + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickX, clickY)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + iosDriver.perform(Arrays.asList(tap)); + } + + +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnTextOccurrenceBased.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnTextOccurrenceBased.java new file mode 100644 index 00000000..c352d357 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/ClickOnTextOccurrenceBased.java @@ -0,0 +1,134 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; +import java.util.List; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +@Data +@Action(actionText = "Tap on text name, occurrence position found-at-position", + description = "Tap on given text and at given occurrence based on text coordinates", + applicationType = ApplicationType.IOS) + +public class ClickOnTextOccurrenceBased extends IOSAction { + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestData(reference = "name") + private com.testsigma.sdk.TestData text; + + @TestData(reference = "found-at-position") + private com.testsigma.sdk.TestData position; + + @Override + protected Result execute() { + Result result = Result.SUCCESS; + try { + IOSDriver iosDriver = (IOSDriver) this.driver; + File baseImageFile = ((TakesScreenshot) iosDriver).getScreenshotAs(OutputType.FILE); + BufferedImage bufferedImage = ImageIO.read(baseImageFile); + int imageWidth = bufferedImage.getWidth(); + int imageHeight = bufferedImage.getHeight(); + logger.info("Width of image: " + imageWidth); + logger.info("Height of image: " + imageHeight); + Dimension dimension = iosDriver.manage().window().getSize(); + int screenWidth = dimension.width; + int screenHeight = dimension.height; + logger.info("Screen width: " + screenWidth); + logger.info("Screen height: " + screenHeight); + List textPoints = ocr.extractTextFromPage(); + int target_occurrence = Integer.parseInt(position.getValue().toString()); + printAllCoordinates(textPoints); + OCRTextPoint textPoint = getTextPointFromText(textPoints, target_occurrence); + if (textPoint == null) { + result = Result.FAILED; + setErrorMessage("Given text is not found"); + + } else { + logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + + ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2()); + logger.info("Performing click.."); + int x = (textPoint.getX1() + textPoint.getX2()) / 2; + int y = (textPoint.getY1() + textPoint.getY2()) / 2; + logger.info("Screen shot based click locations: x="+x+"y="+y); + double xRelative = ((double) x / imageWidth); + double yRelative = ((double) y / imageHeight); + logger.info("Error ratios: x relative: "+xRelative+" y relative: "+yRelative); + int clickLocationX; + int clickLocationY; + if(Math.abs(imageWidth-screenWidth) > 20){ + clickLocationX = (int) (xRelative * screenWidth); + clickLocationY = (int) (yRelative * screenHeight); + } else { + clickLocationX = x; + clickLocationY = y; + } + logger.info("Actual Click locations: x="+clickLocationX+"y="+clickLocationY); + clickOnCoordinates(clickLocationX, clickLocationY); + setSuccessMessage("Click operation performed on the text " + + " Text coordinates :" + "x1-" + textPoint.getX1() + ", x2-" + textPoint.getX2() + ", y1-" + textPoint.getY1() + ", y2-" + textPoint.getY2()); + } + } catch(Exception e) { + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } + private OCRTextPoint getTextPointFromText(List textPoints,int target_occurrence) { + if(textPoints == null) { + return null; + } + int occurrences = 0; + for(OCRTextPoint textPoint: textPoints) { + if(text.getValue().equals(textPoint.getText())) { + occurrences+=1; + if(occurrences == target_occurrence){ + return textPoint; + } + } + } + return null; + } + + private void printAllCoordinates(List textPoints) { + for(OCRTextPoint textPoint: textPoints) { + logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n"); + } + } + public void clickOnCoordinates(int clickX, int clickY) { + + IOSDriver iosDriver = (IOSDriver) this.driver; + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickX, clickY)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + iosDriver.perform(Arrays.asList(tap)); + } + + + +} + diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImage.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImage.java new file mode 100644 index 00000000..4558a42c --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImage.java @@ -0,0 +1,54 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.IOSAction; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; + +import java.io.File; + +@Data +@Action(actionText = "Verify if image image-file present in current-page", + description = "Verify if the given image is present in current page", + applicationType = ApplicationType.IOS) +public class SearchImage extends IOSAction { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + IOSDriver iosDriver = (IOSDriver) this.driver; + Result result = Result.SUCCESS; + File baseImageFile= ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage(testData1.getValue().toString()); + if(response.getIsFound()){ + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + + } else { + setErrorMessage("Given image not found in the current page"); + result = Result.FAILED; + } + return result; + } + +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithScale.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithScale.java new file mode 100644 index 00000000..6039c2b1 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithScale.java @@ -0,0 +1,57 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.IOSAction; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; + +import java.io.File; + +@Data +@Action(actionText = "Verify if image image-url is present in current-page with search threshold threshold-value (Ex: 0.9 , means 90% match) and scale as scale-factor", + description = "Verify if the given image with threshold and scale is present in the current page", + applicationType = ApplicationType.IOS) +public class SearchImageWithScale extends IOSAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + @TestData(reference = "scale-factor") + private com.testsigma.sdk.TestData testData3; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + IOSDriver iosDriver = (IOSDriver) this.driver; + Result result = Result.SUCCESS; + File baseImageFile= ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString()), Integer.parseInt(testData3.getValue().toString())); + if(response.getIsFound()){ + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + + } else { + setErrorMessage("Given image not found in the current page"); + result = Result.FAILED; + } + return result; + + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithThreshold.java new file mode 100644 index 00000000..64c2642b --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/ios/SearchImageWithThreshold.java @@ -0,0 +1,55 @@ +package com.testsigma.addons.ios; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.IOSAction; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.ios.IOSDriver; +import lombok.Data; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; + +import java.io.File; + +@Data +@Action(actionText = "Verify if image image-url is present in current-page with search threshold threshold-value (Ex: 0.9 , means 90% match)", + description = "Verify if the give image with threshold is present in current page", + applicationType = ApplicationType.IOS) +public class SearchImageWithThreshold extends IOSAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + IOSDriver iosDriver = (IOSDriver) this.driver; + Result result = Result.SUCCESS; + File baseImageFile= ((TakesScreenshot)iosDriver).getScreenshotAs(OutputType.FILE); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + FindImageResponse response = ocr.findImage(testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString())); + if(response.getIsFound()){ + setSuccessMessage("Image Found :" + response.getIsFound() + + " Image coordinates :" + "x1-" + response.getX1() + ", x2-" + response.getX2() + ", y1-" + response.getY1() + ", y2-" + response.getY2()); + + } else { + setErrorMessage("Given image not found in the current page"); + result = Result.FAILED; + } + return result; + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java new file mode 100644 index 00000000..d6542c03 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImage.java @@ -0,0 +1,65 @@ +package com.testsigma.addons.mobile_web; + +import com.google.common.collect.ImmutableMap; +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.WebAction; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.AppiumDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.remote.DriverCommand; + +import java.io.File; + +import static com.testsigma.addons.mobile_web.util.ClickActionUtils.performClick; +import static com.testsigma.addons.mobile_web.util.ContextUtils.getCurrentContext; + +@Data +@Action(actionText = "Click on image image-url", + description = "Click on given image", + applicationType = ApplicationType.MOBILE_WEB) +public class ClickOnImage extends WebAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + AppiumDriver appiumDriver = (AppiumDriver) driver; + String existingContext = getCurrentContext(appiumDriver); + try { + TakesScreenshot scrShot = ((TakesScreenshot) driver); + File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE); + + // Switch to native app context + appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", "NATIVE_APP")); + + String successMessage= performClick(ocr, testData1.getValue().toString(), null, + baseImageFile,testStepResult.getScreenshotUrl(), appiumDriver, logger); + + setSuccessMessage(successMessage); + + // Switch back to old context + appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + + return Result.SUCCESS; + } catch (Exception e) { + appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + logger.info("Exception: " + ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click on image: " + e.getMessage()); + return Result.FAILED; + } + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java new file mode 100644 index 00000000..a9be241f --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/ClickOnImageWithThreshold.java @@ -0,0 +1,71 @@ +package com.testsigma.addons.mobile_web; + +import com.google.common.collect.ImmutableMap; +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.WebAction; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import io.appium.java_client.AppiumDriver; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.NoSuchElementException; +import org.openqa.selenium.OutputType; +import org.openqa.selenium.TakesScreenshot; +import org.openqa.selenium.remote.DriverCommand; + +import java.io.File; + +import static com.testsigma.addons.mobile_web.util.ClickActionUtils.performClick; +import static com.testsigma.addons.mobile_web.util.ContextUtils.getCurrentContext; + +@Data +@Action(actionText = "Click on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match)", + description = "Click on given image with threshold", + applicationType = ApplicationType.MOBILE_WEB) +public class ClickOnImageWithThreshold extends WebAction { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + AppiumDriver appiumDriver = (AppiumDriver) driver; + String existingContext = getCurrentContext(appiumDriver); + try { + TakesScreenshot scrShot = ((TakesScreenshot) driver); + File baseImageFile = scrShot.getScreenshotAs(OutputType.FILE); + + // Switch to native app context + appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", "NATIVE_APP")); + + String successMessage = performClick(ocr, testData1.getValue().toString(), + Float.valueOf(testData2.getValue().toString()), baseImageFile, testStepResult.getScreenshotUrl(), + appiumDriver, logger); + + setSuccessMessage(successMessage); + + // Switch back to old context + appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + + return Result.SUCCESS; + } catch (Exception e) { + appiumDriver.execute(DriverCommand.SWITCH_TO_CONTEXT, ImmutableMap.of("name", existingContext)); + logger.info("Exception: " + ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click on image: " + e.getMessage()); + return Result.FAILED; + } + } + +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImage.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImage.java new file mode 100644 index 00000000..11f27b07 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImage.java @@ -0,0 +1,34 @@ +package com.testsigma.addons.mobile_web; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.openqa.selenium.NoSuchElementException; + +@Data +@Action(actionText = "Verify if image image-file present in current-page", + description = "Verify if the given image is present in current page", + applicationType = ApplicationType.MOBILE_WEB) +public class SearchImage extends com.testsigma.addons.web.SearchImage { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + super.setTestData1(testData1); + super.setOcr(ocr); + super.setTestStepResult(testStepResult); + return super.execute(); + } + +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithScale.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithScale.java new file mode 100644 index 00000000..2e4c97e3 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithScale.java @@ -0,0 +1,40 @@ +package com.testsigma.addons.mobile_web; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.openqa.selenium.NoSuchElementException; + +@Data +@Action(actionText = "Verify if image image-url is present in current-page with search threshold threshold-value (Ex: 0.9 , means 90% match) and scale as scale-factor", + description = "Verify if the given image with threshold and scale is present in the current page", + applicationType = ApplicationType.MOBILE_WEB) +public class SearchImageWithScale extends com.testsigma.addons.web.SearchImageWithScale { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + @TestData(reference = "scale-factor") + private com.testsigma.sdk.TestData testData3; + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + super.setTestData1(testData1); + super.setTestData2(testData2); + super.setTestData3(testData3); + super.setOcr(ocr); + super.setTestStepResult(testStepResult); + return super.execute(); + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithThreshold.java new file mode 100644 index 00000000..170c4c87 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/SearchImageWithThreshold.java @@ -0,0 +1,37 @@ +package com.testsigma.addons.mobile_web; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.openqa.selenium.NoSuchElementException; + +@Data +@Action(actionText = "Verify if image image-url is present in current-page with search threshold threshold-value (Ex: 0.9 , means 90% match)", + description = "Verify if the give image with threshold is present in current page", + applicationType = ApplicationType.MOBILE_WEB) +public class SearchImageWithThreshold extends com.testsigma.addons.web.SearchImageWithThreshold { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + super.setTestData1(testData1); + super.setTestData2(testData2); + super.setOcr(ocr); + super.setTestStepResult(testStepResult); + return super.execute(); + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ClickActionUtils.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ClickActionUtils.java new file mode 100644 index 00000000..a6dd0996 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ClickActionUtils.java @@ -0,0 +1,109 @@ +package com.testsigma.addons.mobile_web.util; + +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.Logger; +import com.testsigma.sdk.OCR; +import io.appium.java_client.AppiumDriver; +import org.openqa.selenium.Dimension; +import org.openqa.selenium.interactions.Pause; +import org.openqa.selenium.interactions.PointerInput; +import org.openqa.selenium.interactions.Sequence; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.File; +import java.io.IOException; +import java.util.Collections; + +import static java.time.Duration.ofMillis; +import static org.openqa.selenium.interactions.PointerInput.Kind.TOUCH; +import static org.openqa.selenium.interactions.PointerInput.MouseButton.LEFT; +import static org.openqa.selenium.interactions.PointerInput.Origin.viewport; + +public class ClickActionUtils { + private static void performClickWithFinger(int clickLocationX, int clickLocationY, AppiumDriver appiumDriver) + throws Exception { + try { + PointerInput FINGER = new PointerInput(TOUCH, "finger"); + Sequence tap = new Sequence(FINGER, 1) + .addAction(FINGER.createPointerMove(ofMillis(0), viewport(), clickLocationX, clickLocationY)) + .addAction(FINGER.createPointerDown(LEFT.asArg())) + .addAction(new Pause(FINGER, ofMillis(2))) + .addAction(FINGER.createPointerUp(LEFT.asArg())); + appiumDriver.perform(Collections.singletonList(tap)); + } catch (Exception e) { + throw new Exception(e); + } + } + + public static String performClick(OCR ocr, String targetImgUrl, Float threshold, + File baseImage, String screenshotUrl, AppiumDriver driver, + Logger logger) throws Exception { + + try { + FindImageResponse response = null; + if(threshold == null){ + response = findImageAndGetResponse(ocr, targetImgUrl, baseImage, screenshotUrl); + } else { + response = findImageAndGetResponse(ocr, targetImgUrl, baseImage, screenshotUrl, threshold); + } + + + if (response.getIsFound()) { + BufferedImage image = ImageIO.read(baseImage); + int x1 = response.getX1(); + int x2 = response.getX2(); + int y1 = response.getY1(); + int y2 = response.getY2(); + + int imageWidth = image.getWidth(); + int imageHeight = image.getHeight(); + logger.info("imageWidth: " + imageWidth); + logger.info("imageHeight: " + imageHeight); + + int xMean = ((x1 + x2) / 2); + int yMean = ((y1 + y2) / 2); + + double xRelative = (double) xMean / imageWidth; + double yRelative = (double) yMean / imageHeight; + logger.info("xRelative: " + xRelative); + logger.info("yRelative: " + yRelative); + + Dimension screenDimension = driver.manage().window().getSize(); + logger.info("screenDimension: " + screenDimension); + + int clickLocationX = (int) (xRelative * screenDimension.getWidth()); + int clickLocationY = (int) (yRelative * screenDimension.getHeight()); + logger.info(String.format("Click location: (%d,%d)", clickLocationX, clickLocationY)); + + performClickWithFinger(clickLocationX,clickLocationY, driver); + return String.format("Image Found: %s; Image coordinates: x1-%d, x2-%d, y1-%d, y2-%d", + response.getIsFound(), x1, x2, y1, y2); + } else{ + throw new Exception("Unable to fetch the coordinates"); + } + + } catch (IOException e) { + throw new Exception(e); + } + } + + + private static FindImageResponse findImageAndGetResponse(OCR ocr, String targetImgUrl, File baseImage, + String screenshotUrl) { + // upload base image to url + ocr.uploadFile(screenshotUrl, baseImage); + + //find image + return ocr.findImage(targetImgUrl); + } + + private static FindImageResponse findImageAndGetResponse(OCR ocr, String targetImgUrl, File baseImage, + String screenshotUrl, Float threshold) { + // upload base image to url + ocr.uploadFile(screenshotUrl, baseImage); + + //find image + return ocr.findImage(targetImgUrl, threshold); + } +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java new file mode 100644 index 00000000..8a5db32b --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/mobile_web/util/ContextUtils.java @@ -0,0 +1,37 @@ +package com.testsigma.addons.mobile_web.util; + +import com.google.common.collect.ImmutableMap; +import io.appium.java_client.AppiumDriver; +import org.openqa.selenium.WebDriverException; +import org.openqa.selenium.remote.DriverCommand; +import org.openqa.selenium.remote.Response; + +import java.util.LinkedHashSet; +import java.util.Set; + +public class ContextUtils { + public static Set getContextHandles(AppiumDriver driver) { + Response response = driver.execute(DriverCommand.GET_CONTEXT_HANDLES, ImmutableMap.of()); + Object value = response.getValue(); + try { + //noinspection unchecked + java.util.List returnedValues = (java.util.List) value; + return new LinkedHashSet<>(returnedValues); + } catch (ClassCastException ex) { + throw new WebDriverException( + "Returned value cannot be converted to List: " + value, ex); + } + } + + public static String getCurrentContext(AppiumDriver driver) { + Response response = driver.execute(DriverCommand.GET_CURRENT_CONTEXT_HANDLE, ImmutableMap.of()); + Object value = response.getValue(); + try { + //noinspection unchecked + return (String) value; + } catch (ClassCastException ex) { + throw new WebDriverException( + "Returned value cannot be converted to List: " + value, ex); + } + } +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImage.java index 8a688781..0979697e 100644 --- a/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImage.java +++ b/image_based_actions/src/main/java/com/testsigma/addons/web/ClickOnImage.java @@ -22,7 +22,7 @@ @Data @Action(actionText = "Click on image image-url", - description = "Click on give image", + description = "Click on given image", applicationType = ApplicationType.WEB, useCustomScreenshot = true) public class ClickOnImage extends WebAction { diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImage.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImage.java new file mode 100644 index 00000000..0532cb16 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImage.java @@ -0,0 +1,101 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.WindowsAction; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.NoSuchElementException; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.event.InputEvent; +import java.awt.image.BufferedImage; +import java.io.File; + +@Data +@Action(actionText = "Double-Click on image image-url", + description = "Double Click on give image", + applicationType = ApplicationType.WINDOWS) +public class DoubleClickOnImage extends WindowsAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + // Instantiate the Robot Class + Robot robot = new Robot(); + + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + logger.info("Amazon s3 url in which we are storing base image"+url); + ocr.uploadFile(url, baseImageFile); + logger.info("url: "+ testStepResult.getScreenshotUrl()); + FindImageResponse responseObject = ocr.findImage(testData1.getValue().toString()); + logger.info("Image search Response : "+responseObject); + if (responseObject.getIsFound()){ + boolean isFound = responseObject.getIsFound(); + int x1 = responseObject.getX1(); + int y1 = responseObject.getY1(); + int x2 = responseObject.getX2(); + int y2 = responseObject.getY2(); + + int clickLocationX = (x1 + x2) / 2; + int clickLocationY = (y1 + y2) / 2; + + logger.info("Click Location X: " + clickLocationX); + logger.info("Click Location Y: " + clickLocationY); + + robot.mouseMove(clickLocationX, clickLocationY); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + setSuccessMessage("Image Found :" + isFound + + " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2); + Thread.sleep(1000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageOccurrenceBased.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageOccurrenceBased.java new file mode 100644 index 00000000..7ab32963 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageOccurrenceBased.java @@ -0,0 +1,102 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.WindowsAction; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.NoSuchElementException; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.event.InputEvent; +import java.awt.image.BufferedImage; +import java.io.File; + +@Data +@Action(actionText = "Double-Click on image image-url, occurrence position found-at-position", + description = "Double Click on give image, at given position", + applicationType = ApplicationType.WINDOWS) +public class DoubleClickOnImageOccurrenceBased extends WindowsAction { + @TestData(reference = "image-url") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "found-at-position") + private com.testsigma.sdk.TestData testData2; + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + // Instantiate the Robot Class + Robot robot = new Robot(); + + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + ocr.uploadFile(url, baseImageFile); + logger.info("url: "+ testStepResult.getScreenshotUrl()); + int occurrence = Integer.parseInt(testData2.getValue().toString()); + FindImageResponse responseObject = ocr.findImage(testData1.getValue().toString(),occurrence); + if (responseObject.getIsFound()){ + boolean isFound = responseObject.getIsFound(); + int x1 = responseObject.getX1(); + int y1 = responseObject.getY1(); + int x2 = responseObject.getX2(); + int y2 = responseObject.getY2(); + + int clickLocationX = (x1 + x2) / 2; + int clickLocationY = (y1 + y2) / 2; + + logger.info("Click Location X: " + clickLocationX); + logger.info("Click Location Y: " + clickLocationY); + + robot.mouseMove(clickLocationX, clickLocationY); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + setSuccessMessage("Image Found :" + isFound + + " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2); + Thread.sleep(1000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } + +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThreshold.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThreshold.java new file mode 100644 index 00000000..8b44f14c --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThreshold.java @@ -0,0 +1,101 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.WindowsAction; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.NoSuchElementException; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.event.InputEvent; +import java.awt.image.BufferedImage; +import java.io.File; + +@Data +@Action(actionText = "Double-Click on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match)", + description = "Double Click on given image with threshold", + applicationType = ApplicationType.WINDOWS) +public class DoubleClickOnImageWithThreshold extends WindowsAction { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + + + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + Robot robot = new Robot(); + + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + ocr.uploadFile(url, baseImageFile); + logger.info("url: "+ testStepResult.getScreenshotUrl()); + FindImageResponse responseObject = ocr.findImage(testData1.getValue().toString(), Float.valueOf(testData2.getValue().toString())); + if (responseObject.getIsFound()){ + boolean isFound = responseObject.getIsFound(); + int x1 = responseObject.getX1(); + int y1 = responseObject.getY1(); + int x2 = responseObject.getX2(); + int y2 = responseObject.getY2(); + + int clickLocationX = (x1 + x2) / 2; + int clickLocationY = (y1 + y2) / 2; + + logger.info("Click Location X: " + clickLocationX); + logger.info("Click Location Y: " + clickLocationY); + + robot.mouseMove(clickLocationX, clickLocationY); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + setSuccessMessage("Image Found :" + isFound + + " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2); + Thread.sleep(2000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } +} \ No newline at end of file diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThresholdOccurrenceBased.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThresholdOccurrenceBased.java new file mode 100644 index 00000000..9c41a8ea --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnImageWithThresholdOccurrenceBased.java @@ -0,0 +1,106 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.ApplicationType; +import com.testsigma.sdk.FindImageResponse; +import com.testsigma.sdk.Result; +import com.testsigma.sdk.WindowsAction; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; +import org.apache.commons.lang3.exception.ExceptionUtils; +import org.openqa.selenium.NoSuchElementException; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.event.InputEvent; +import java.awt.image.BufferedImage; +import java.io.File; + +@Data +@Action(actionText = "Double-Click on image image-file with applied search threshold threshold-value (Ex: 0.9 , means 90% match), occurrence position found-at-position", + description = "Double Click on given image with threshold at the given position", + applicationType = ApplicationType.WINDOWS) +public class DoubleClickOnImageWithThresholdOccurrenceBased extends WindowsAction { + @TestData(reference = "image-file") + private com.testsigma.sdk.TestData testData1; + @TestData(reference = "threshold-value") + private com.testsigma.sdk.TestData testData2; + + @TestData(reference = "found-at-position") + private com.testsigma.sdk.TestData testData3; + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() throws NoSuchElementException { + Result result = Result.SUCCESS; + try { + // Instantiate the Robot Class + Robot robot = new Robot(); + + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + ocr.uploadFile(url, baseImageFile); + logger.info("url: "+ testStepResult.getScreenshotUrl()); + Float threshold = Float.valueOf(testData2.getValue().toString()); + int occurrence = Integer.parseInt(testData3.getValue().toString()); + FindImageResponse responseObject = ocr.findImage(testData1.getValue().toString(),occurrence,threshold); + if (responseObject.getIsFound()){ + boolean isFound = responseObject.getIsFound(); + int x1 = responseObject.getX1(); + int y1 = responseObject.getY1(); + int x2 = responseObject.getX2(); + int y2 = responseObject.getY2(); + + int clickLocationX = (x1 + x2) / 2; + int clickLocationY = (y1 + y2) / 2; + + logger.info("Click Location X: " + clickLocationX); + logger.info("Click Location Y: " + clickLocationY); + + robot.mouseMove(clickLocationX, clickLocationY); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + setSuccessMessage("Image Found :" + isFound + + " Image coordinates :" + "x1-" + x1 + ", x2-" + x2 + ", y1-" + y1 + ", y2-" + y2); + Thread.sleep(2000); + } else { + setErrorMessage("Unable to fetch the coordinates"); + result = Result.FAILED; + } + } + catch (Exception e){ + logger.info("Exception: "+ ExceptionUtils.getStackTrace(e)); + setErrorMessage("Exception occurred while performing click action"); + result = Result.FAILED; + } + return result; + } + +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnText.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnText.java new file mode 100644 index 00000000..25719521 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnText.java @@ -0,0 +1,151 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.event.InputEvent; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; +import java.util.List; + +@Data +@Action(actionText = "Double-Click on text name", + description = "Click on the text using the text coordinates", + applicationType = ApplicationType.WINDOWS, + useCustomScreenshot = true) + +public class DoubleClickOnText extends WindowsAction { + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestData(reference = "name") + private com.testsigma.sdk.TestData text; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() { + Result result = Result.SUCCESS; + try{ + Robot robot = new Robot(); + + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + OCRImage ocrImage = new OCRImage(); + ocrImage.setOcrImageFile(baseImageFile); + + List textPoints = ocr.extractTextFromImage(ocrImage); + printAllCoordinates(textPoints); + OCRTextPoint textPoint = getTextPointFromText(textPoints); + if(textPoint == null) { + result = Result.FAILED; + setErrorMessage("Given text is not found"); + + } else { + logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + + ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2()); + doubleClickOnCoordinates(textPoint); + tmp = robot.createScreenCapture(screenSize); + filename = "screenshot"+System.currentTimeMillis()+".jpg"; + path = tempDir + filename; + ImageIO.write(tmp, "jpg",new File(path)); + baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + ocr.uploadFile(url, baseImageFile); + setSuccessMessage("Click operation performed on the text " + + " Text coordinates :" + "x1-" + textPoint.getX1() + ", x2-" + textPoint.getX2() + ", y1-" + textPoint.getY1() + ", y2-" + textPoint.getY2()); + } + } catch (Exception e){ + logger.info("Exception: "+ Arrays.toString(e.getStackTrace())); + setErrorMessage("Exception occurred while searching for the given text"); + result = Result.FAILED; + } + + + return result; + } + private OCRTextPoint getTextPointFromText(List textPoints) { + if(textPoints == null) { + return null; + } + for(OCRTextPoint textPoint: textPoints) { + if(text.getValue().equals(textPoint.getText())) { + return textPoint; + + } + } + return null; + } + + private void printAllCoordinates(List textPoints) { + for(OCRTextPoint textPoint: textPoints) { + logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n"); + } + } + public void clickOnCoordinates(OCRTextPoint textPoint) throws AWTException { + Robot robot = new Robot(); + + int x1 = textPoint.getX1(); + int y1 = textPoint.getY1(); + int x2 = textPoint.getX2(); + int y2 = textPoint.getY2(); + + int x = (x1 + x2) / 2; + int y = (y1 + y2) / 2; + + logger.info("MEAN X coordinate: " + x + "\n"); + logger.info("MEAN Y coordinate: " + y + "\n"); + + robot.mouseMove(x, y); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + } + public void doubleClickOnCoordinates(OCRTextPoint textPoint) throws AWTException { + Robot robot = new Robot(); + + int x1 = textPoint.getX1(); + int y1 = textPoint.getY1(); + int x2 = textPoint.getX2(); + int y2 = textPoint.getY2(); + + int x = (x1 + x2) / 2; + int y = (y1 + y2) / 2; + + logger.info("MEAN X coordinate: " + x + "\n"); + logger.info("MEAN Y coordinate: " + y + "\n"); + + robot.mouseMove(x, y); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + } + + +} diff --git a/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnTextOccurrenceBased.java b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnTextOccurrenceBased.java new file mode 100644 index 00000000..1026cf25 --- /dev/null +++ b/image_based_actions/src/main/java/com/testsigma/addons/windows/DoubleClickOnTextOccurrenceBased.java @@ -0,0 +1,160 @@ +package com.testsigma.addons.windows; + +import com.testsigma.sdk.*; +import com.testsigma.sdk.annotation.Action; +import com.testsigma.sdk.annotation.OCR; +import com.testsigma.sdk.annotation.TestData; +import com.testsigma.sdk.annotation.TestStepResult; +import lombok.Data; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.event.InputEvent; +import java.awt.image.BufferedImage; +import java.io.File; +import java.util.Arrays; +import java.util.List; + +@Data +@Action(actionText = "Double-Click on text name, occurrence position found-at-position", + description = "Click on given text and at given occurrence based on text coordinates", + applicationType = ApplicationType.WINDOWS, + useCustomScreenshot = true) + +public class DoubleClickOnTextOccurrenceBased extends WindowsAction { + @OCR + private com.testsigma.sdk.OCR ocr; + + @TestData(reference = "name") + private com.testsigma.sdk.TestData text; + + @TestData(reference = "found-at-position") + private com.testsigma.sdk.TestData position; + + @TestStepResult + private com.testsigma.sdk.TestStepResult testStepResult; + + @Override + protected Result execute() { + Result result = Result.SUCCESS; + try{ + Robot robot = new Robot(); + + + // Fetch the Details of the Screen Size + Rectangle screenSize = new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()); + + // Take the Snapshot of the Screen + BufferedImage tmp = robot.createScreenCapture(screenSize); + + // Provide the destination details to copy the screenshot + String tempDir = System.getProperty("java.io.tmpdir"); + String filename = "screenshot"+System.currentTimeMillis()+".jpg"; + String path = tempDir + filename; + + // To copy source image in to destination path + ImageIO.write(tmp, "jpg",new File(path)); + + + int width = tmp.getWidth(); + int height = tmp.getHeight(); + logger.info("Width of image: " + width); + logger.info("Height of image: " + height); + + File baseImageFile = new File(path); + OCRImage ocrImage = new OCRImage(); + ocrImage.setOcrImageFile(baseImageFile); + + List textPoints = ocr.extractTextFromImage(ocrImage); + int target_occurrence = Integer.parseInt(position.getValue().toString()); + printAllCoordinates(textPoints); + OCRTextPoint textPoint = getTextPointFromText(textPoints,target_occurrence); + if(textPoint == null) { + result = Result.FAILED; + setErrorMessage("Given text is not found"); + + } else { + logger.info("Found Textpoint with text = " + textPoint.getText() + ", x1 = " + textPoint.getX1() + + ", y1 = " + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 = " + textPoint.getY2()); + doubleClickOnCoordinates(textPoint); + tmp = robot.createScreenCapture(screenSize); + filename = "screenshot"+System.currentTimeMillis()+".jpg"; + path = tempDir + filename; + ImageIO.write(tmp, "jpg",new File(path)); + baseImageFile = new File(path); + String url = testStepResult.getScreenshotUrl(); + ocr.uploadFile(url, baseImageFile); + setSuccessMessage("Click operation performed on the text " + + " Text coordinates :" + "x1-" + textPoint.getX1() + ", x2-" + textPoint.getX2() + ", y1-" + textPoint.getY1() + ", y2-" + textPoint.getY2()); + } + } catch (Exception e){ + logger.info("Exception: "+ Arrays.toString(e.getStackTrace())); + setErrorMessage("Exception occurred while searching for the given text"); + result = Result.FAILED; + } + + + return result; + } + private OCRTextPoint getTextPointFromText(List textPoints,int target_occurrence) { + if(textPoints == null) { + return null; + } + int occurrences = 0; + for(OCRTextPoint textPoint: textPoints) { + if(text.getValue().equals(textPoint.getText())) { + occurrences+=1; + if(occurrences == target_occurrence){ + return textPoint; + } + } + } + return null; + } + + private void printAllCoordinates(List textPoints) { + for(OCRTextPoint textPoint: textPoints) { + logger.info("text =" + textPoint.getText() + "x1 = " + textPoint.getX1() + ", y1 =" + textPoint.getY1() + ", x2 = " + textPoint.getX2() + ", y2 =" + textPoint.getY2() +"\n\n\n\n"); + } + } + public void clickOnCoordinates(OCRTextPoint textPoint) throws AWTException { + Robot robot = new Robot(); + + int x1 = textPoint.getX1(); + int y1 = textPoint.getY1(); + int x2 = textPoint.getX2(); + int y2 = textPoint.getY2(); + + int x = (x1 + x2) / 2; + int y = (y1 + y2) / 2; + + logger.info("MEAN X coordinate: " + x + "\n"); + logger.info("MEAN Y coordinate: " + y + "\n"); + + robot.mouseMove(x, y); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + } + public void doubleClickOnCoordinates(OCRTextPoint textPoint) throws AWTException { + Robot robot = new Robot(); + + int x1 = textPoint.getX1(); + int y1 = textPoint.getY1(); + int x2 = textPoint.getX2(); + int y2 = textPoint.getY2(); + + int x = (x1 + x2) / 2; + int y = (y1 + y2) / 2; + + logger.info("MEAN X coordinate: " + x + "\n"); + logger.info("MEAN Y coordinate: " + y + "\n"); + + robot.mouseMove(x, y); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + robot.mousePress(InputEvent.BUTTON1_DOWN_MASK); + robot.mouseRelease(InputEvent.BUTTON1_DOWN_MASK); + } + +} +