From 24c19a2a6eb84f222c645d930934ea04a7bbbb5f Mon Sep 17 00:00:00 2001 From: Jonathan Asai Date: Sun, 3 Nov 2024 11:07:14 -0800 Subject: [PATCH 1/3] Add files via upload Added a simple on/off cursor when the thumb is extended on the right hand. --- main.py | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 1c3a1f2..7053f18 100644 --- a/main.py +++ b/main.py @@ -42,21 +42,31 @@ def findPosition(self, img, handNo=0, draw=True): cx, cy = int(lm.x * w), int(lm.y * h) lmlist.append([id, cx, cy]) if draw: - cv2.circle(img, (cx, cy), 3, (255, 0, 255), cv2.FILLED) + #cv2.circle(img, (cx, cy), 3, (255, 0, 255), cv2.FILLED) + cv2.putText(img,str(int(id)),(cx,cy),cv2.FONT_HERSHEY_PLAIN,1,(255,0,255),1) return lmlist # Draws the axis for vector graphing def axis(img): thickness = 3 - cv2.line(img, (img.shape[1]-DIM, 0), (img.shape[1]-DIM, DIM), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-DIM, DIM), (img.shape[1], DIM), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-(DIM*2), 0), (img.shape[1]-(DIM*2), DIM), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-(DIM*2), DIM), (img.shape[1]-DIM, DIM), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-(DIM*2), DIM), (img.shape[1]-(DIM*2), (DIM*2)), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-(DIM*2), (DIM*2)), (img.shape[1]-DIM, (DIM*2)), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-DIM, DIM), (img.shape[1]-DIM, (DIM*2)), (0, 0, 255), thickness) - cv2.line(img, (img.shape[1]-DIM, (DIM*2)), (img.shape[1], (DIM*2)), (0, 0, 255), thickness) + color = (0, 0, 255) + cv2.line(img, (img.shape[1]-DIM, 0), (img.shape[1]-DIM, DIM), color, thickness) + cv2.line(img, (img.shape[1]-DIM, DIM), (img.shape[1], DIM), color, thickness) + cv2.line(img, (img.shape[1]-(DIM*2), 0), (img.shape[1]-(DIM*2), DIM), color, thickness) + cv2.line(img, (img.shape[1]-(DIM*2), DIM), (img.shape[1]-DIM, DIM), color, thickness) + cv2.line(img, (img.shape[1]-(DIM*2), DIM), (img.shape[1]-(DIM*2), (DIM*2)), color, thickness) + cv2.line(img, (img.shape[1]-(DIM*2), (DIM*2)), (img.shape[1]-DIM, (DIM*2)), color, thickness) + cv2.line(img, (img.shape[1]-DIM, DIM), (img.shape[1]-DIM, (DIM*2)), color, thickness) + cv2.line(img, (img.shape[1]-DIM, (DIM*2)), (img.shape[1], (DIM*2)), color, thickness) + return img + +def cursor(img,points): + thickness = 1 + offset = 15 + color = (0, 255, 0) + cv2.line(img, (points[1]-offset, points[2]), (points[1]+offset,points[2]), color, thickness) + cv2.line(img, (points[1], points[2]-offset), (points[1], points[2]+offset), color, thickness) return img def main(): @@ -77,7 +87,6 @@ def main(): # vectors is x distance from origin, y distance from origin, angle made success, img = cap.read() vecs = np.array([img.shape[1]-DIM,DIM,0], dtype=float) - while True: #get success, img = cap.read() @@ -87,8 +96,16 @@ def main(): img = axis(img) img = detector.findHands(img) lmlist = detector.findPosition(img) + + #Gesture detection if len(lmlist) >= 8: - print(lmlist[8]) + #Detects if the thumb is in or out + if lmlist[4][1]<=lmlist[3][1]: + # Activates cursor if the thumb is out + img = cursor(img,lmlist[8]) + + #if len(lmlist) >= 8: + # print(lmlist[8]) # calculate the frames cTime = time.time() @@ -103,6 +120,7 @@ def main(): cv2.putText(img, "Settings", (10, 50), cv2.FONT_HERSHEY_PLAIN, 3, (255, 0, 255), 3) cv2.putText(img, str(int(fps)),(10, img.shape[0] - 30), cv2.FONT_HERSHEY_PLAIN, 3, (255,0, 255),3) cv2.imshow("Image", img) + # When you press q it quits the program if cv2.waitKey(10) & 0xFF == ord('q'): break From e11d5e34ec48d18655b47f8c305a7face3ec121c Mon Sep 17 00:00:00 2001 From: Jonathan Asai Date: Wed, 20 Nov 2024 20:36:16 -0800 Subject: [PATCH 2/3] Add files via upload --- main.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index 7053f18..bbab3c8 100644 --- a/main.py +++ b/main.py @@ -2,6 +2,7 @@ import cv2 import time import mediapipe as mp +import pyautogui # Globals DIM = 250 @@ -72,12 +73,21 @@ def cursor(img,points): def main(): pTime = 0 cTime = 0 + #Timers for the timed functions + clickTimer = 0 + keyTimer = 0 # this initalizes what camera to use if an external camera is not used it switches to laptop cam cap = cv2.VideoCapture(1) if not cap.isOpened(): cap.release() cap = cv2.VideoCapture(0) + frame_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + frame_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + screen_Width, screen_Height = pyautogui.size() + xScale = screen_Width/frame_width + yScale = screen_Height/frame_height + # hand detection class detector = handDetector() @@ -97,12 +107,54 @@ def main(): img = detector.findHands(img) lmlist = detector.findPosition(img) - #Gesture detection + #Gesture detection block + #thumb, index, middle, ring, and pinkie are true if they are extended and false if they aren't + thumb = False + index = False + middle = False + ring = False + pinkie = False + if len(lmlist) >= 8: - #Detects if the thumb is in or out - if lmlist[4][1]<=lmlist[3][1]: - # Activates cursor if the thumb is out - img = cursor(img,lmlist[8]) + if lmlist[4][1]=3: + pyautogui.click() + clickTimer = 0 + else : + clickTimer=0 + + #if the index, middle, and ring are up for 3 seconds then toggles the windows onscreen keyboard + if index and middle and ring: + if keyTimer == 0: + keyTimer=time.time() + elif time.time()-keyTimer >=3: + pyautogui.hotkey('ctrl','win', 'o') + keyTimer = 0 + else : + keyTimer = 0 + + # stops the program if the index and pinkie are out and no other fingers are up + if (index and pinkie) and not (thumb or middle or ring): + break + #if len(lmlist) >= 8: # print(lmlist[8]) From 30e4cc025ff99adca678c94ef7547ca5ba405dcb Mon Sep 17 00:00:00 2001 From: Jonathan Asai Date: Sat, 23 Nov 2024 20:16:08 -0800 Subject: [PATCH 3/3] Add files via upload --- README.md | 1 + main.py | 115 +++++++++++++++++++++++++++++++++++++----------------- 2 files changed, 80 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index d536f72..b1b34be 100644 --- a/README.md +++ b/README.md @@ -1 +1,2 @@ # VectorCalc +# Uses mediapipe, pyauto, and a few other libraries to perform hand gesture recognition diff --git a/main.py b/main.py index bbab3c8..e478f04 100644 --- a/main.py +++ b/main.py @@ -49,17 +49,16 @@ def findPosition(self, img, handNo=0, draw=True): # Draws the axis for vector graphing -def axis(img): +def axis(img,x,y,width,length): thickness = 3 color = (0, 0, 255) - cv2.line(img, (img.shape[1]-DIM, 0), (img.shape[1]-DIM, DIM), color, thickness) - cv2.line(img, (img.shape[1]-DIM, DIM), (img.shape[1], DIM), color, thickness) - cv2.line(img, (img.shape[1]-(DIM*2), 0), (img.shape[1]-(DIM*2), DIM), color, thickness) - cv2.line(img, (img.shape[1]-(DIM*2), DIM), (img.shape[1]-DIM, DIM), color, thickness) - cv2.line(img, (img.shape[1]-(DIM*2), DIM), (img.shape[1]-(DIM*2), (DIM*2)), color, thickness) - cv2.line(img, (img.shape[1]-(DIM*2), (DIM*2)), (img.shape[1]-DIM, (DIM*2)), color, thickness) - cv2.line(img, (img.shape[1]-DIM, DIM), (img.shape[1]-DIM, (DIM*2)), color, thickness) - cv2.line(img, (img.shape[1]-DIM, (DIM*2)), (img.shape[1], (DIM*2)), color, thickness) + xOffset = int(width) + yOffset = int(length) + #Middle axis + cv2.line(img, ((x-xOffset)//2,y//2), ((x+xOffset)//2,y//2), color, thickness) + cv2.line(img, (x//2,(y-yOffset)//2), (x//2,(y+yOffset)//2), color, thickness) + #Outer box + cv2.rectangle(img, ((x-xOffset)//2,(y-yOffset)//2), ((x+xOffset)//2,(y+yOffset)//2), color, thickness) return img def cursor(img,points): @@ -70,12 +69,28 @@ def cursor(img,points): cv2.line(img, (points[1], points[2]-offset), (points[1], points[2]+offset), color, thickness) return img +def countdown(img,timer,x,y): + color = (255,255,255) + size = 1 + if timer - time.time() >=2: + color = (255,0,0) + size = 2 + elif timer - time.time() >=1: + color = (0,255,0) + size = 3 + elif timer - time.time() >=0: + color = (0,0,255) + size = 4 + cv2.putText(img, str(int(timer - time.time())),(x, y), cv2.FONT_HERSHEY_PLAIN, 2, color,size) + def main(): pTime = 0 cTime = 0 #Timers for the timed functions clickTimer = 0 keyTimer = 0 + modeTimer = 0 + stopTimer = 0 # this initalizes what camera to use if an external camera is not used it switches to laptop cam cap = cv2.VideoCapture(1) @@ -87,10 +102,11 @@ def main(): screen_Width, screen_Height = pyautogui.size() xScale = screen_Width/frame_width yScale = screen_Height/frame_height - + # hand detection class detector = handDetector() + mouse = False # the first vector is the origin vector # vectors will be formed from the orgin to any place in the graph @@ -103,7 +119,7 @@ def main(): if not success: break img = cv2.flip(img, 1) - img = axis(img) + img = axis(img,frame_width,frame_height,100*xScale,100*yScale) img = detector.findHands(img) lmlist = detector.findPosition(img) @@ -116,7 +132,7 @@ def main(): pinkie = False if len(lmlist) >= 8: - if lmlist[4][1]lmlist[3][1] and lmlist[1][1]>lmlist[0][1]): thumb = True if lmlist[8][2]=3: - pyautogui.click() - clickTimer = 0 + if mouse: + #mouse mode + + #If the thumb and index finger are extended, it moves the cursor to the fingertip + if thumb and index: + cursor(img,lmlist[8]) + pyautogui.moveTo(lmlist[8][1]*xScale, lmlist[8][2]*yScale) + #if the middle finger is also extended for 3 seconds, clicks where the cursor is + if middle and not (ring or pinkie): + countdown(img,clickTimer,lmlist[8][1],lmlist[8][2]+20) + if clickTimer == 0: + clickTimer=time.time() + 3 + elif clickTimer-time.time() <=0: + pyautogui.click() + clickTimer=time.time() + 3 + else : + clickTimer=time.time() + 3 + + #if only the index, middle, and ring are up for 3 seconds then toggles the windows onscreen keyboard + if not thumb and not pinkie and index and middle and ring: + countdown(img,keyTimer,lmlist[12][1],lmlist[12][2]+20) + if keyTimer == 0: + keyTimer = time.time() + elif keyTimer - time.time() <=0: + pyautogui.hotkey('ctrl','win', 'o') + keyTimer=time.time() + 3 else : - clickTimer=0 - - #if the index, middle, and ring are up for 3 seconds then toggles the windows onscreen keyboard - if index and middle and ring: - if keyTimer == 0: - keyTimer=time.time() - elif time.time()-keyTimer >=3: - pyautogui.hotkey('ctrl','win', 'o') - keyTimer = 0 - else : - keyTimer = 0 - + keyTimer=time.time() + 3 + elif vector: + #vector mode + #implimentation of vector modes + tmp = 0 + # stops the program if the index and pinkie are out and no other fingers are up if (index and pinkie) and not (thumb or middle or ring): - break + countdown(img,stopTimer,frame_width-50,frame_height-50) + if stopTimer - time.time() <=0: + break + else : + stopTimer=time.time() + 3 + #if len(lmlist) >= 8: @@ -173,6 +213,9 @@ def main(): cv2.putText(img, str(int(fps)),(10, img.shape[0] - 30), cv2.FONT_HERSHEY_PLAIN, 3, (255,0, 255),3) cv2.imshow("Image", img) + #generates a cursor + + # When you press q it quits the program if cv2.waitKey(10) & 0xFF == ord('q'): break