-
Notifications
You must be signed in to change notification settings - Fork 1
/
sift.py
145 lines (112 loc) · 6.17 KB
/
sift.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
import cv2
import numpy as np
import mouse
from scipy.spatial.distance import pdist
sift = cv2.SIFT_create()
#feature matching
class SIFT:
def __init__(self,reference:str,arrow_f:str,arrow_b:str) -> None:
self.reference = cv2.imread(reference)
self.arrow_f = cv2.imread(arrow_f)
self.arrow_b = cv2.imread(arrow_b)
self.output_img = None
def show_SIFT_features(self):
cv2.imshow('SIFT', self.output_img)
cv2.waitKey(0)
def get_SIFT_features(self,img):
#create matcher
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=True)
#img = cv2.imread(image)
red = img[:,:,0] #cross or star
green = img[:,:,1] #arrow front
blue = img[:,:,2] #arrow back
#red = cv2.cvtColor(red, cv2.COLOR_BGR2GRAY)
#cv2.imshow('red',red)
#cv2.waitKey(0)
#cv2.imshow('green',green)
#cv2.waitKey(0)
try:
keypoints_ref, descriptors_ref = sift.detectAndCompute(self.reference,None)
keypoints_red, descriptors_red = sift.detectAndCompute(red,None)
#Get matches between reference image and red symbol
matches = bf.match(descriptors_ref,descriptors_red)
matches = sorted(matches,key=lambda x:x.distance)
# Get the indices of the matching keypoints
matches_red = [match.trainIdx for match in matches]
matches_ref = [match.queryIdx for match in matches]
# Get the coordinates of the matching keypoints
matches_red = [keypoints_red[idx].pt for idx in matches_red]
matches_ref = [keypoints_ref[idx].pt for idx in matches_ref]
# Find the average x and y coordinates of the matches
avg_x_red = np.mean([pt[0] for pt in matches_red])
avg_y_red = np.mean([pt[1] for pt in matches_red])
avg_x_ref = np.mean([pt[0] for pt in matches_ref])
avg_y_ref = np.mean([pt[1] for pt in matches_ref])
# Combine the average x and y coordinates into a tuple
avg_coord_red = (avg_x_red, avg_y_red)
avg_coord_ref = (avg_x_ref, avg_y_ref)
#print(avg_coord_red,avg_coord_ref)
cross_diff = abs(avg_coord_ref[0] - 43.6)
star_diff = abs(avg_coord_ref[0] - 117.6)
if cross_diff < star_diff: #cross, test for up and left
# green = cv2.cvtColor(green, cv2.COLOR_BGR2GRAY)
keypoints_arrow_f, descriptors_arrow_f = sift.detectAndCompute(self.arrow_f,None)
keypoints_green, descriptors_green = sift.detectAndCompute(green,None)
matches_arrow = bf.match(descriptors_arrow_f,descriptors_green)
matches_arrow = sorted(matches_arrow, key = lambda x:x.distance)
# Get the indices of the matching keypoints in the green channel
matches_green = [match.trainIdx for match in matches_arrow]
# Get the coordinates of the matching keypoints in the green channel
matches_green_coords = [keypoints_green[idx].pt for idx in matches_green]
img = cv2.drawMatches(self.arrow_f, keypoints_arrow_f, green, keypoints_green, matches_arrow[0:10], None, flags=2)
#cv2.imshow('arrow',img)
#cv2.waitKey(0)
# Get the average coordinates of the matching keypoints in the green channel
avg_x_green = np.mean([pt[0] for pt in matches_green_coords])
avg_y_green = np.mean([pt[1] for pt in matches_green_coords])
#print(avg_x_green,avg_y_green)
if abs(avg_x_green-avg_x_red) > abs(avg_y_green-avg_y_red):
#arrow to the left of the cross
#move mouse to the left
print('left')
mouse.move(-10, 0, absolute=False, duration=0.02)
else:
#arrow above the cross
#move mouse up
print(abs(avg_y_green - avg_y_red))
if(abs(avg_y_green - avg_y_red) > 60):
print('up')
mouse.move(0, -10, absolute=False, duration=0.02)
else:
print('click')
mouse.click()
else:
#it's a star, test for down and right directions
keypoints_arrow_b, descriptors_arrow_b = sift.detectAndCompute(self.arrow_b,None)
keypoints_green, descriptors_green = sift.detectAndCompute(green,None)
matches_arrow = bf.match(descriptors_arrow_b,descriptors_green)
matches_arrow = sorted(matches_arrow, key = lambda x:x.distance)
# Get the indices of the matching keypoints in the green channel
matches_green = [match.trainIdx for match in matches_arrow]
# Get the coordinates of the matching keypoints in the green channel
matches_green_coords = [keypoints_green[idx].pt for idx in matches_green]
img = cv2.drawMatches(self.arrow_b, keypoints_arrow_b, green, keypoints_green, matches_arrow[0:10], None, flags=2)
#cv2.imshow('arrow',img)
#cv2.waitKey(0)
# Get the average coordinates of the matching keypoints in the green channel
avg_x_green = np.mean([pt[0] for pt in matches_green_coords])
avg_y_green = np.mean([pt[1] for pt in matches_green_coords])
#print(avg_x_green,avg_y_green)
if abs(avg_x_green-avg_x_red) > abs(avg_y_green-avg_y_red):
#arrow to the right of the cross
#move mouse to the right
print('right')
mouse.move(10, 0, absolute=False, duration=0.02)
else:
#arrow below the cross
#move mouse down
print('down')
mouse.move(0, 10, absolute=False, duration=0.02)
self.output_img = cv2.drawMatches(self.reference, keypoints_ref, red, keypoints_red, matches[0:10], None, flags=2)
except:
print("no hand")