Skip to content

Commit bece24a

Browse files
gorvingorvin
committed
New screen related method implementation featuring direct opencv call.
1 parent 039c936 commit bece24a

File tree

1 file changed

+106
-104
lines changed

1 file changed

+106
-104
lines changed

‎selenium-bravo-servlet/src/main/java/com/bravostudiodev/selenium/SikuliScreen.java‎

Lines changed: 106 additions & 104 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.sikuli.api.*;
66
import org.sikuli.api.robot.desktop.DesktopKeyboard;
77
import org.sikuli.api.robot.desktop.DesktopMouse;
8+
import org.sikuli.api.robot.desktop.DesktopScreen;
89

910
import javax.imageio.ImageIO;
1011
import javax.xml.bind.DatatypeConverter;
@@ -31,28 +32,32 @@ public SikuliScreen() {
3132

3233
}
3334

34-
private static String objectSerialize(Object obj) throws IOException {
35+
private static String objectSerialize(Object obj) {
3536
try (final ByteArrayOutputStream byout = new ByteArrayOutputStream();
3637
final ObjectOutputStream out = new ObjectOutputStream(byout)
3738
) {
3839
out.writeObject(obj);
3940
LOGGER.info("Serialized data");
4041
return DatatypeConverter.printBase64Binary(byout.toByteArray());
42+
} catch(IOException e) {
43+
throw new ObjectSerializeException("Failed object serialization");
4144
}
4245
}
4346

44-
private static Object objectParse(String objStr) throws IOException, ClassNotFoundException {
47+
private static Object objectParse(String objStr) {
4548
byte[] objBytes = DatatypeConverter.parseBase64Binary(objStr);
4649

4750
try (final ByteArrayInputStream byin = new ByteArrayInputStream(objBytes);
4851
final ObjectInputStream in = new ObjectInputStream(byin)
4952
) {
5053
LOGGER.info("Parsing data...");
5154
return in.readObject();
55+
} catch(IOException | ClassNotFoundException e) {
56+
throw new ObjectParseException("Failed object parsing");
5257
}
5358
}
5459

55-
public static ScreenRegion getScreenRegion(int x, int y, int w, int h) {
60+
private static ScreenRegion getScreenRegion(int x, int y, int w, int h) {
5661
ScreenRegion primary = new DesktopScreenRegion();
5762
return primary.getRelativeScreenRegion(x, y, w, h);
5863
}
@@ -91,122 +96,133 @@ public String takeBase64Screenshot(java.awt.Rectangle rc) throws IOException {
9196
return imageToBase64(region.capture());
9297
}
9398

94-
public void drag() {
95-
new DesktopMouse().drag(new DesktopScreenRegion().getCenter());
99+
public String getScreenRectangle() {
100+
return objectSerialize(new DesktopScreen(0).getBounds());
96101
}
97102

98-
public void drag(int xOff, int yOff) {
99-
new DesktopMouse().drag(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
103+
public String getScreenDimensions() {
104+
return objectSerialize(new DesktopScreen(0).getSize());
100105
}
101106

102-
public void drag(ScreenRegion region) {
107+
private void drag(java.awt.Rectangle rc) {
108+
if(null == rc)
109+
return;
110+
ScreenRegion region = getScreenRegion(rc);
111+
if(null == region)
112+
return;
103113
new DesktopMouse().drag(region.getCenter());
104114
}
105115

106-
public void drag(ScreenRegion region, int xOff, int yOff) {
107-
new DesktopMouse().drag(region.getCenter().getRelativeScreenLocation(xOff, yOff));
108-
}
109-
110-
public void drop() {
111-
new DesktopMouse().click(new DesktopScreenRegion().getCenter());
112-
}
113-
114-
public void drop(int xOff, int yOff) {
115-
new DesktopMouse().drop(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
116+
private void drop(java.awt.Rectangle rc) {
117+
if(null == rc)
118+
return;
119+
ScreenRegion region = getScreenRegion(rc);
120+
if(null == region)
121+
return;
122+
new DesktopMouse().drop(region.getCenter());
116123
}
117124

118-
public void drop(ScreenRegion region) {
119-
new DesktopMouse().drop(region.getCenter());
125+
private void move(java.awt.Rectangle rc) {
126+
if(null == rc)
127+
return;
128+
ScreenRegion region = getScreenRegion(rc);
129+
if(null == region)
130+
return;
131+
new DesktopMouse().move(region.getCenter());
120132
}
121133

122-
public void drop(ScreenRegion region, int xOff, int yOff) {
123-
new DesktopMouse().drop(region.getCenter().getRelativeScreenLocation(xOff, yOff));
134+
private void hover(java.awt.Rectangle rc) {
135+
if(null == rc)
136+
return;
137+
ScreenRegion region = getScreenRegion(rc);
138+
if(null == region)
139+
return;
140+
new DesktopMouse().hover(region.getCenter());
124141
}
125142

126-
public void click() {
127-
new DesktopMouse().click(new DesktopScreenRegion().getCenter());
143+
private void click(java.awt.Rectangle rc) {
144+
if(null == rc)
145+
return;
146+
ScreenRegion region = getScreenRegion(rc);
147+
if(null == region)
148+
return;
149+
new DesktopMouse().click(region.getCenter());
128150
}
129151

130-
public void click(int xOff, int yOff) {
131-
new DesktopMouse().click(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
152+
private void doubleClick(java.awt.Rectangle rc) {
153+
if(null == rc)
154+
return;
155+
ScreenRegion region = getScreenRegion(rc);
156+
if(null == region)
157+
return;
158+
new DesktopMouse().doubleClick(region.getCenter());
132159
}
133160

134-
public void click(ScreenRegion region) {
135-
ScreenLocation loc = region.getCenter();
136-
LOGGER.info("clicking region");
137-
new DesktopMouse().click(loc);
161+
private void rightClick(java.awt.Rectangle rc) {
162+
if(null == rc)
163+
return;
164+
ScreenRegion region = getScreenRegion(rc);
165+
if(null == region)
166+
return;
167+
new DesktopMouse().rightClick(region.getCenter());
138168
}
139169

140-
public void click(ScreenRegion region, int xOff, int yOff) {
141-
new DesktopMouse().click(region.getCenter().getRelativeScreenLocation(xOff, yOff));
170+
public void drag(String rcB64) {
171+
drag((java.awt.Rectangle) objectParse(rcB64));
142172
}
143173

144-
public void rightClick() {
145-
new DesktopMouse().rightClick(new DesktopScreenRegion().getCenter());
174+
public void drop(String rcB64) {
175+
drop((java.awt.Rectangle) objectParse(rcB64));
146176
}
147177

148-
public void rightClick(int xOff, int yOff) {
149-
new DesktopMouse().rightClick(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
178+
public void move(String rcB64) {
179+
move((java.awt.Rectangle) objectParse(rcB64));
150180
}
151181

152-
public void rightClick(ScreenRegion region) {
153-
new DesktopMouse().rightClick(region.getCenter());
182+
public void hover(String rcB64) {
183+
hover((java.awt.Rectangle) objectParse(rcB64));
154184
}
155185

156-
public void rightClick(ScreenRegion region, int xOff, int yOff) {
157-
new DesktopMouse().rightClick(region.getCenter().getRelativeScreenLocation(xOff, yOff));
186+
public void click(String rcB64) {
187+
click((java.awt.Rectangle) objectParse(rcB64));
158188
}
159189

160-
public void doubleClick() {
161-
new DesktopMouse().doubleClick(new DesktopScreenRegion().getCenter());
190+
public void doubleClick(String rcB64) {
191+
doubleClick((java.awt.Rectangle) objectParse(rcB64));
162192
}
163193

164-
public void doubleClick(int xOff, int yOff) {
165-
new DesktopMouse().doubleClick(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
194+
public void rightClick(String rcB64) {
195+
rightClick((java.awt.Rectangle) objectParse(rcB64));
166196
}
167197

168-
public void doubleClick(ScreenRegion region) {
169-
new DesktopMouse().doubleClick(region.getCenter());
198+
public void drag() {
199+
new DesktopMouse().drag(new DesktopScreenRegion().getCenter());
170200
}
171201

172-
public void doubleClick(ScreenRegion region, int xOff, int yOff) {
173-
new DesktopMouse().doubleClick(region.getCenter().getRelativeScreenLocation(xOff, yOff));
202+
public void drop() {
203+
new DesktopMouse().click(new DesktopScreenRegion().getCenter());
174204
}
175205

176-
public void hover() {
177-
new DesktopMouse().hover(new DesktopScreenRegion().getCenter());
206+
public void click() {
207+
new DesktopMouse().click(new DesktopScreenRegion().getCenter());
178208
}
179209

180-
@SuppressWarnings("unused")
181-
public void hover(int xOff, int yOff) {
182-
new DesktopMouse().hover(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
210+
public void rightClick() {
211+
new DesktopMouse().rightClick(new DesktopScreenRegion().getCenter());
183212
}
184213

185-
public void hover(ScreenRegion region) {
186-
new DesktopMouse().hover(region.getCenter());
214+
public void doubleClick() {
215+
new DesktopMouse().doubleClick(new DesktopScreenRegion().getCenter());
187216
}
188217

189-
@SuppressWarnings("unused")
190-
public void hover(ScreenRegion region, int xOff, int yOff) {
191-
new DesktopMouse().hover(region.getCenter().getRelativeScreenLocation(xOff, yOff));
218+
public void hover() {
219+
new DesktopMouse().hover(new DesktopScreenRegion().getCenter());
192220
}
193221

194222
public void move() {
195223
new DesktopMouse().move(new DesktopScreenRegion().getCenter());
196224
}
197225

198-
public void move(int xOff, int yOff) {
199-
new DesktopMouse().move(new DesktopScreenRegion().getCenter().getRelativeScreenLocation(xOff, yOff));
200-
}
201-
202-
public void move(ScreenRegion region) {
203-
new DesktopMouse().move(region.getCenter());
204-
}
205-
206-
public void move(ScreenRegion region, int xOff, int yOff) {
207-
new DesktopMouse().move(region.getCenter().getRelativeScreenLocation(xOff, yOff));
208-
}
209-
210226
public void press() {
211227
new DesktopMouse().press();
212228
}
@@ -295,31 +311,17 @@ public UnknownImageTarget(String msg) {
295311
}
296312
}
297313

298-
// public ScreenRegion find(String targetName) {
299-
// LOGGER.info("finding target " + targetName);
300-
// ImageTarget targetImg = images.get(targetName);
301-
// if (null == targetImg)
302-
// throw new UnknownImageTarget("Unkown target name " + targetName + ", not added with addTarget method");
303-
// try {
304-
// return new DesktopScreenRegion().find(targetImg);
305-
// } catch (Exception e) {
306-
// LOGGER.log(Level.WARNING, "Didn't find image " + targetName);
307-
// throw e;
308-
// }
309-
// }
310-
//
311-
// public List<ScreenRegion> findAll(String targetName) {
312-
// LOGGER.info("finding all targets " + targetName);
313-
// ImageTarget targetImg = images.get(targetName);
314-
// if (null == targetImg)
315-
// throw new UnknownImageTarget("Unkown target name " + targetName + ", not added with addTarget method");
316-
// try {
317-
// return new DesktopScreenRegion().findAll(targetImg);
318-
// } catch (Exception e) {
319-
// LOGGER.log(Level.WARNING, "Didn't find image " + targetName);
320-
// throw e;
321-
// }
322-
// }
314+
public static class ObjectParseException extends RuntimeException {
315+
public ObjectParseException(String msg) {
316+
super(msg);
317+
}
318+
}
319+
320+
public static class ObjectSerializeException extends RuntimeException {
321+
public ObjectSerializeException(String msg) {
322+
super(msg);
323+
}
324+
}
323325

324326
private static byte[] getPngBytes(BufferedImage biImg) throws IOException {
325327
try (final ByteArrayOutputStream os = new ByteArrayOutputStream()) {
@@ -328,16 +330,11 @@ private static byte[] getPngBytes(BufferedImage biImg) throws IOException {
328330
}
329331
}
330332

331-
public static ArrayList<java.awt.Rectangle> findAll(BufferedImage biFull, BufferedImage biTpl, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
333+
private static ArrayList<java.awt.Rectangle> findAll(BufferedImage biFull, BufferedImage biTpl, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
332334
assert 0.0 <= MIN_SCORE && MIN_SCORE <= 1.0;
333335

334-
// new Integer(b64PNG.hashCode()).toString();
335-
336336
byte[] byFull = getPngBytes(biFull);
337337
byte[] byTpl = getPngBytes(biTpl);
338-
// Mat imgSrcColor = imread(pathImage);
339-
// Mat imgSrcGray = new Mat(imgSrcColor.size(), CV_8UC1);
340-
// cvtColor(imgSrcColor, imgSrcGray, COLOR_BGR2GRAY);
341338
Mat imgSrcGray = imdecode(new Mat(byFull), CV_LOAD_IMAGE_GRAYSCALE);
342339
Mat imgTplGray = imdecode(new Mat(byTpl), CV_LOAD_IMAGE_GRAYSCALE);
343340
Size size = new Size(imgSrcGray.cols() - imgTplGray.cols() + 1, imgSrcGray.rows() - imgTplGray.rows() + 1);
@@ -366,18 +363,23 @@ public static ArrayList<java.awt.Rectangle> findAll(BufferedImage biFull, Buffer
366363
return lMatches;
367364
}
368365

369-
public static java.awt.Rectangle findMostSimilar(BufferedImage biFull, BufferedImage biTpl, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
366+
private static java.awt.Rectangle findMostSimilar(BufferedImage biFull, BufferedImage biTpl, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
370367
ArrayList<java.awt.Rectangle> lMatches = findAll(biFull, biTpl, MIN_SCORE, fullDim);
371368
return lMatches.isEmpty() ? null : lMatches.get(0); // first match has highest similarity
372369
}
373370

374-
public String findAllOnScreen(String targetName, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
371+
public String allOnScreenTargets(String targetName, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
375372
LOGGER.info("finding target " + targetName);
376373
ImageTarget targetImg = images.get(targetName);
377374
if (null == targetImg)
378375
throw new UnknownImageTarget("Unkown target name " + targetName + ", not added with addTarget method");
379376
ScreenRegion primary = new DesktopScreenRegion();
380-
381377
return objectSerialize(findAll(primary.capture(), targetImg.getImage(), MIN_SCORE, fullDim));
382378
}
379+
380+
public String allOnScreenB64Images(String b64Image, double MIN_SCORE, java.awt.Dimension fullDim) throws IOException {
381+
LOGGER.info("finding target image from base64 string");
382+
ScreenRegion primary = new DesktopScreenRegion();
383+
return objectSerialize(findAll(primary.capture(), imageFromBase64(b64Image), MIN_SCORE, fullDim));
384+
}
383385
}

0 commit comments

Comments
 (0)