update

Author: harshbg

Code/Install_Packages.txt

@@ -0,0 +1,7 @@
+h5py
+numpy
+scikit-learn
+sklearn
+keras
+opencv-python
+pyttsx3

Code/Install_Packages_gpu.txt

@@ -0,0 +1,8 @@
+h5py
+numpy
+scikit-learn
+sklearn
+tensorflow-gpu
+keras
+opencv-python
+pyttsx3

Code/Rotate_images.py

@@ -0,0 +1,17 @@
+import cv2, os
+
+def flip_images():
+	gest_folder = "gestures"
+	images_labels = []
+	images = []
+	labels = []
+	for g_id in os.listdir(gest_folder):
+		for i in range(1200):
+			path = gest_folder+"/"+g_id+"/"+str(i+1)+".jpg"
+			new_path = gest_folder+"/"+g_id+"/"+str(i+1+1200)+".jpg"
+			print(path)
+			img = cv2.imread(path, 0)
+			img = cv2.flip(img, 1)
+			cv2.imwrite(new_path, img)
+
+flip_images()

Code/cnn_model_train.py

@@ -0,0 +1,76 @@
+import numpy as np
+import pickle
+import cv2, os
+from glob import glob
+from keras import optimizers
+from keras.models import Sequential
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import Flatten
+from keras.layers.convolutional import Conv2D
+from keras.layers.convolutional import MaxPooling2D
+from keras.utils import np_utils
+from keras.callbacks import ModelCheckpoint
+from keras import backend as K
+K.set_image_dim_ordering('tf')
+
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+
+def get_image_size():
+	img = cv2.imread('gestures/1/100.jpg', 0)
+	return img.shape
+
+def get_num_of_classes():
+	return len(glob('gestures/*'))
+
+image_x, image_y = get_image_size()
+
+def cnn_model():
+	num_of_classes = get_num_of_classes()
+	model = Sequential()
+	model.add(Conv2D(16, (2,2), input_shape=(image_x, image_y, 1), activation='relu'))
+	model.add(MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same'))
+	model.add(Conv2D(32, (3,3), activation='relu'))
+	model.add(MaxPooling2D(pool_size=(3, 3), strides=(3, 3), padding='same'))
+	model.add(Conv2D(64, (5,5), activation='relu'))
+	model.add(MaxPooling2D(pool_size=(5, 5), strides=(5, 5), padding='same'))
+	model.add(Flatten())
+	model.add(Dense(128, activation='relu'))
+	model.add(Dropout(0.2))
+	model.add(Dense(num_of_classes, activation='softmax'))
+	sgd = optimizers.SGD(lr=1e-2)
+	model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
+	filepath="cnn_model_keras2.h5"
+	checkpoint1 = ModelCheckpoint(filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')
+	callbacks_list = [checkpoint1]
+	#from keras.utils import plot_model
+	#plot_model(model, to_file='model.png', show_shapes=True)
+	return model, callbacks_list
+
+def train():
+	with open("train_images", "rb") as f:
+		train_images = np.array(pickle.load(f))
+	with open("train_labels", "rb") as f:
+		train_labels = np.array(pickle.load(f), dtype=np.int32)
+
+	with open("val_images", "rb") as f:
+		val_images = np.array(pickle.load(f))
+	with open("val_labels", "rb") as f:
+		val_labels = np.array(pickle.load(f), dtype=np.int32)
+
+	train_images = np.reshape(train_images, (train_images.shape[0], image_x, image_y, 1))
+	val_images = np.reshape(val_images, (val_images.shape[0], image_x, image_y, 1))
+	train_labels = np_utils.to_categorical(train_labels)
+	val_labels = np_utils.to_categorical(val_labels)
+
+	print(val_labels.shape)
+
+	model, callbacks_list = cnn_model()
+	model.summary()
+	model.fit(train_images, train_labels, validation_data=(val_images, val_labels), epochs=15, batch_size=500, callbacks=callbacks_list)
+	scores = model.evaluate(val_images, val_labels, verbose=0)
+	print("CNN Error: %.2f%%" % (100-scores[1]*100))
+	#model.save('cnn_model_keras2.h5')
+
+train()
+K.clear_session();

Code/create_gestures.py

@@ -0,0 +1,106 @@
+import cv2
+import numpy as np
+import pickle, os, sqlite3, random
+
+image_x, image_y = 50, 50
+
+def get_hand_hist():
+	with open("hist", "rb") as f:
+		hist = pickle.load(f)
+	return hist
+
+def init_create_folder_database():
+	# create the folder and database if not exist
+	if not os.path.exists("gestures"):
+		os.mkdir("gestures")
+	if not os.path.exists("gesture_db.db"):
+		conn = sqlite3.connect("gesture_db.db")
+		create_table_cmd = "CREATE TABLE gesture ( g_id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, g_name TEXT NOT NULL )"
+		conn.execute(create_table_cmd)
+		conn.commit()
+
+def create_folder(folder_name):
+	if not os.path.exists(folder_name):
+		os.mkdir(folder_name)
+
+def store_in_db(g_id, g_name):
+	conn = sqlite3.connect("gesture_db.db")
+	cmd = "INSERT INTO gesture (g_id, g_name) VALUES (%s, \'%s\')" % (g_id, g_name)
+	try:
+		conn.execute(cmd)
+	except sqlite3.IntegrityError:
+		choice = input("g_id already exists. Want to change the record? (y/n): ")
+		if choice.lower() == 'y':
+			cmd = "UPDATE gesture SET g_name = \'%s\' WHERE g_id = %s" % (g_name, g_id)
+			conn.execute(cmd)
+		else:
+			print("Doing nothing...")
+			return
+	conn.commit()
+	
+def store_images(g_id):
+	total_pics = 1200
+	hist = get_hand_hist()
+	cam = cv2.VideoCapture(1)
+	if cam.read()[0]==False:
+		cam = cv2.VideoCapture(0)
+	x, y, w, h = 300, 100, 300, 300
+
+	create_folder("gestures/"+str(g_id))
+	pic_no = 0
+	flag_start_capturing = False
+	frames = 0
+	
+	while True:
+		img = cam.read()[1]
+		img = cv2.flip(img, 1)
+		imgHSV = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
+		dst = cv2.calcBackProject([imgHSV], [0, 1], hist, [0, 180, 0, 256], 1)
+		disc = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(10,10))
+		cv2.filter2D(dst,-1,disc,dst)
+		blur = cv2.GaussianBlur(dst, (11,11), 0)
+		blur = cv2.medianBlur(blur, 15)
+		thresh = cv2.threshold(blur,0,255,cv2.THRESH_BINARY+cv2.THRESH_OTSU)[1]
+		thresh = cv2.merge((thresh,thresh,thresh))
+		thresh = cv2.cvtColor(thresh, cv2.COLOR_BGR2GRAY)
+		thresh = thresh[y:y+h, x:x+w]
+		contours = cv2.findContours(thresh.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)[1]
+
+		if len(contours) > 0:
+			contour = max(contours, key = cv2.contourArea)
+			if cv2.contourArea(contour) > 10000 and frames > 50:
+				x1, y1, w1, h1 = cv2.boundingRect(contour)
+				pic_no += 1
+				save_img = thresh[y1:y1+h1, x1:x1+w1]
+				if w1 > h1:
+					save_img = cv2.copyMakeBorder(save_img, int((w1-h1)/2) , int((w1-h1)/2) , 0, 0, cv2.BORDER_CONSTANT, (0, 0, 0))
+				elif h1 > w1:
+					save_img = cv2.copyMakeBorder(save_img, 0, 0, int((h1-w1)/2) , int((h1-w1)/2) , cv2.BORDER_CONSTANT, (0, 0, 0))
+				save_img = cv2.resize(save_img, (image_x, image_y))
+				rand = random.randint(0, 10)
+				if rand % 2 == 0:
+					save_img = cv2.flip(save_img, 1)
+				cv2.putText(img, "Capturing...", (30, 60), cv2.FONT_HERSHEY_TRIPLEX, 2, (127, 255, 255))
+				cv2.imwrite("gestures/"+str(g_id)+"/"+str(pic_no)+".jpg", save_img)
+
+		cv2.rectangle(img, (x,y), (x+w, y+h), (0,255,0), 2)
+		cv2.putText(img, str(pic_no), (30, 400), cv2.FONT_HERSHEY_TRIPLEX, 1.5, (127, 127, 255))
+		cv2.imshow("Capturing gesture", img)
+		cv2.imshow("thresh", thresh)
+		keypress = cv2.waitKey(1)
+		if keypress == ord('c'):
+			if flag_start_capturing == False:
+				flag_start_capturing = True
+			else:
+				flag_start_capturing = False
+				frames = 0
+		if flag_start_capturing == True:
+			frames += 1
+		if pic_no == total_pics:
+			break
+
+init_create_folder_database()
+g_id = input("Enter gesture no.: ")
+g_name = input("Enter gesture name/text: ")
+store_in_db(g_id, g_name)
+store_images(g_id)

Code/display_gestures.py

@@ -0,0 +1,42 @@
+import cv2, os, random
+import numpy as np
+
+def get_image_size():
+	img = cv2.imread('gestures/0/100.jpg', 0)
+	return img.shape
+
+gestures = os.listdir('gestures/')
+gestures.sort(key = int)
+begin_index = 0
+end_index = 5
+image_x, image_y = get_image_size()
+
+if len(gestures)%5 != 0:
+	rows = int(len(gestures)/5)+1
+else:
+	rows = int(len(gestures)/5)
+
+full_img = None
+for i in range(rows):
+	col_img = None
+	for j in range(begin_index, end_index):
+		img_path = "gestures/%s/%d.jpg" % (j, random.randint(1, 1200))
+		img = cv2.imread(img_path, 0)
+		if np.any(img == None):
+			img = np.zeros((image_y, image_x), dtype = np.uint8)
+		if np.any(col_img == None):
+			col_img = img
+		else:
+			col_img = np.hstack((col_img, img))
+
+	begin_index += 5
+	end_index += 5
+	if np.any(full_img == None):
+		full_img = col_img
+	else:
+		full_img = np.vstack((full_img, col_img))
+
+
+cv2.imshow("gestures", full_img)
+cv2.imwrite('full_img.jpg', full_img)
+cv2.waitKey(0)