Handwriting Recognition in Python

February 5, 2017

Handwriting recognition is a very useful tool in this modern era but can be quite intimidating for many programmers. This post will show you how to create an algorithm to identify characters drawn by the computer mouse.

 

Please Note:

You may need a good understanding of python to fully comprehend this.

 

How it Works

The computer is given a number of examples, say 50, of each character. It then creates a heat map of each character by looking pixel by pixel, and finding the mean number of times each pixel is coloured. The computer stores a large array of the number of times each pixel has been coloured and stores the number of examples used to create the heat map.  We then compare the user's handwriting to each heat map. For every pixel they have coloured, the program takes the average of the averages for each heat map. Whichever heat map scores the highest final average is deemed to be the correct character. This method can only look at one character at time.

 

You will have to add in the 50 (or so) examples yourself.

 

Programming

Let's start by importing some libraries.

 

import pygame
import time
import os

 

Now to set up the window and get the directory path (so we can save files where we like).

 

dir_path = os.path.dirname(os.path.realpath(__file__))

 

pygame.init()

xSize, ySize = 600, 450
screen = pygame.display.set_mode((xSize, ySize))
pygame.display.set_caption("Handwriting Recognition")

 

Here are the variables. 'box' is the side length of the box that you can draw in. The variable 'learnt' is the various characters that the algorithm has been taught to recognize. The variable 'w_char' is the character of the heat map that you are currently adding to. The variable 'bit' stores every heat map in the form of integers. The variable 'num' stores the number of examples each heat map has been given. The variable 'trySum' is used to calculate the percentage accuracy of your handwriting to each heat map. For every pixel you have coloured black, it counts the average number of times that pixel is coloured for different characters shown by the heat maps; 'trySum' is this count. The variable 'acc' stores all of the accuracies calculated for each character.

 

#Variables
box = 100
learnt = ["1","2","3"]
w_char = "3"
bit = {}
num = {}
trySum = {}
acc = {}

 

Now we load the heat maps we have saved.


for item in learnt:
    bit["{0}".format(item)] = []
    num["{0}".format(item)] = 0
    acc["{0}".format(item)] = 0
    trySum["{0}".format(item)] = 0

 

    try:
        with open(dir_path + "/{0}bit".format(item), 'r') as f:
            load = [line.rstrip('\n') for line in f]
        
        for i in load:
            bit["{0}".format(item)].append(int(i))

 

        with open(dir_path + "/{0}bit_num".format(item), 'r') as f:
            num_load = [line.rstrip('\n') for line in f]
            num["{0}".format(item)] = int(num_load[0])

        
    except:
        print("failed to load ", item)

        for i in range(box-1):

            for i in range(box-1):

                bit["{0}".format(item)].append(0)

                num["{0}".format(item)] = 1

 

Here's a short function to clear the screen.

 

#Setup graphics
def setup() :
    screen.fill([255,255,255])
    mx, my = pygame.mouse.get_pos()
    px, py = mx, my
    pygame.draw.lines(screen, [0,0,0], True, [(0,0),(box,0),(box,box),(0,box)])
setup()

 

This is the usual main loop to allow us to update the pygame window frame by frame.

 

#----------------------Main Loop----------------------#

 

clock = pygame.time.Clock()
frameCount = 0
done = False
mouseHold = False

 

while not done:
    mouseClick = False
    mx, my = pygame.mouse.get_pos()
    frameCount += 1
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            done = True
        if event.type == pygame.MOUSEBUTTONDOWN:
            mouseHold = True
            mouseClick = True

        if event.type == pygame.MOUSEBUTTONUP:
            mouseHold = False

 

This code will draw a relatively thick line between the positions of the cursor between frames allowing the user to draw characters in the box.


    #Drawing
    if mouseHold and mx < box and my < box and px < box and py < box:
        pygame.draw.line(screen, [0,0,0], (mx,my), (px,py), 6)
    elif mouseHold and (px < box and py < box) and (mx > box or my > box):
        if mx > box:
            mx = box
        if my > box:
            my = box
        pygame.draw.line(screen, [0,0,0], (mx,my), (px,py), 6)
    elif mouseHold and (mx < box and my < box) and (px > box or py > box):
        if px > box:
            px = box
        if py > box:
            py = box
        pygame.draw.line(screen, [0,0,0], (mx,my), (px,py), 6)
    px, py = mx, my

 

When you press the green button, it adds what you have drawn to the heat map of 'w_char' and saves the heat map; the heat map of 'w_char' is then displayed next to the box. Note 'y*(box-1)+x' finds the correct digit for a particular pixel in a heat map given the x,y coordinate of that pixel.

 

    #Green
    if mouseClick and my > 150 and my < 280 and mx > 10 and mx < 40:
        num["{0}".format(w_char)] += 1
        pygame.draw.rect(screen, [20,150,20], (10, 150, 30, 30)) #Drawing button
        for y in range(box-1):
            for x in range(box-1):
                if screen.get_at((x+1,y+1)) == (0,0,0):
                    bit["{0}".format(w_char)][y*(box-1)+x] += 1

 

        with open(dir_path + "/{0}bit".format(w_char), 'w') as f:
            for s in bit["{0}".format(w_char)]:
                f.write(str(s) + '\n')
                
        with open(dir_path + "/{0}bit_num".format(w_char), 'w') as f:
            f.write(str(num["{0}".format(w_char)]) + '\n')
        
        setup()


        high = max(bit["{0}".format(w_char)])
        if high == 0:
            high = 1
            
        for y in range(box-1):
            for x in range(box-1):
                screen.set_at((x+box+1,y+1),[255*bit["{0}".format(w_char)][y*(box-1)+x]/high,0,0])
                    
        pygame.draw.rect(screen, [20,150,20], (10, 150, 30, 30)) #Drawing button
    else:
        pygame.draw.rect(screen, [20,200,20], (10, 150, 30, 30)) #Drawing button

 

When you press the blue button, it will compare your drawing to all of the saved heat maps and then compare the accuracies calculated.

 

    #Blue
    if mouseClick and my > 150 and my < 280 and mx > 50 and mx < 80:
        pygame.draw.rect(screen, [20,20,150], (50, 150, 30, 30))
        tryNum = 0
        for item in learnt:
            trySum["{0}".format(item)] = 0
        
        for y in range(box-1):
            for x in range(box-1):
                if screen.get_at((x+1,y+1)) == (0,0,0):
                    tryNum += 1
                    for item in learnt:
                        trySum["{0}".format(item)] += bit["{0}".format(item)][y*(box-1)+x]/num["{0}".format(item)]

 

        for item in learnt:
            acc["{0}".format(item)] = 100*trySum["{0}".format(item)]/tryNum

 

        perM = max(acc, key=lambda i: acc[i])
        print(perM)
                    
        pygame.draw.rect(screen, [20,20,150], (50, 150, 30, 30))
    else:
        pygame.draw.rect(screen, [20,20,200], (50, 150, 30, 30))

 

When you press the red button, it will erase what you have drawn.

 

    #Red
    if mouseClick and my > 150 and my < 280 and mx > 90 and mx < 120:
        pygame.draw.rect(screen, [255,255,255], (1, 1, box-1, box-1))
        pygame.draw.rect(screen, [150,20,20], (90, 150, 30, 30))
    else:
        pygame.draw.rect(screen, [200,20,20], (90, 150, 30, 30))

    pygame.display.flip()
    clock.tick(60)

 

pygame.quit()

That is the code complete. Here whole code:

 

import pygame
import time
import os

 

dir_path = os.path.dirname(os.path.realpath(__file__))

 

pygame.init()

xSize, ySize = 600, 450
screen = pygame.display.set_mode((xSize, ySize))
pygame.display.set_caption("Handwriting Recognition")

 

#Variables
box = 100
learnt = ["1","2","3"]
w_char = "3"
bit = {}
num = {}
trySum = {}
acc = {}


for item in learnt:
    bit["{0}".format(item)] = []
    num["{0}".format(item)] = 0
    acc["{0}".format(item)] = 0
    trySum["{0}".format(item)] = 0

 

    try:
        with open(dir_path + "/{0}bit".format(item), 'r') as f:
            load = [line.rstrip('\n') for line in f]
        
        for i in load:
            bit["{0}".format(item)].append(int(i))

 

        with open(dir_path + "/{0}bit_num".format(item), 'r') as f:
            num_load = [line.rstrip('\n') for line in f]
            num["{0}".format(item)] = int(num_load[0])

        
    except:
        print("failed to load ", item)

        for i in range(box-1):

            for i in range(box-1):

                bit["{0}".format(item)].append(0)

                num["{0}".format(item)] = 1

 

#Setup graphics
def setup() :
    screen.fill([255,255,255])
    mx, my = pygame.mouse.get_pos()
    px, py = mx, my
    pygame.draw.lines(screen, [0,0,0], True, [(0,0),(box,0),(box,box),(0,box)])
setup()

 

#----------------------Main Loop----------------------#

 

clock = pygame.time.Clock()
frameCount = 0
done = False
mouseHold = False

 

while not done:
    mouseClick = False
    mx, my = pygame.mouse.get_pos()
    frameCount += 1
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            done = True
        if event.type == pygame.MOUSEBUTTONDOWN:
            mouseHold = True
            mouseClick = True

        if event.type == pygame.MOUSEBUTTONUP:
            mouseHold = False


    #Drawing
    if mouseHold and mx < box and my < box and px < box and py < box:
        pygame.draw.line(screen, [0,0,0], (mx,my), (px,py), 6)
    elif mouseHold and (px < box and py < box) and (mx > box or my > box):
        if mx > box:
            mx = box
        if my > box:
            my = box
        pygame.draw.line(screen, [0,0,0], (mx,my), (px,py), 6)
    elif mouseHold and (mx < box and my < box) and (px > box or py > box):
        if px > box:
            px = box
        if py > box:
            py = box
        pygame.draw.line(screen, [0,0,0], (mx,my), (px,py), 6)
    px, py = mx, my

 

    #Green
    if mouseClick and my > 150 and my < 280 and mx > 10 and mx < 40:
        num["{0}".format(w_char)] += 1
        pygame.draw.rect(screen, [20,150,20], (10, 150, 30, 30)) #Drawing button
        for y in range(box-1):
            for x in range(box-1):
                if screen.get_at((x+1,y+1)) == (0,0,0):
                    bit["{0}".format(w_char)][y*(box-1)+x] += 1

 

        with open(dir_path + "/{0}bit".format(w_char), 'w') as f:
            for s in bit["{0}".format(w_char)]:
                f.write(str(s) + '\n')
                
        with open(dir_path + "/{0}bit_num".format(w_char), 'w') as f:
            f.write(str(num["{0}".format(w_char)]) + '\n')
        
        setup()


        high = max(bit["{0}".format(w_char)])
        if high == 0:
            high = 1
            
        for y in range(box-1):
            for x in range(box-1):
                screen.set_at((x+box+1,y+1),[255*bit["{0}".format(w_char)][y*(box-1)+x]/high,0,0])
                    
        pygame.draw.rect(screen, [20,150,20], (10, 150, 30, 30)) #Drawing button
    else:
        pygame.draw.rect(screen, [20,200,20], (10, 150, 30, 30)) #Drawing button

 

    #Blue
    if mouseClick and my > 150 and my < 280 and mx > 50 and mx < 80:
        pygame.draw.rect(screen, [20,20,150], (50, 150, 30, 30))
        tryNum = 0
        for item in learnt:
            trySum["{0}".format(item)] = 0
        
        for y in range(box-1):
            for x in range(box-1):
                if screen.get_at((x+1,y+1)) == (0,0,0):
                    tryNum += 1
                    for item in learnt:
                        trySum["{0}".format(item)] += bit["{0}".format(item)][y*(box-1)+x]/num["{0}".format(item)]

 

        for item in learnt:
            acc["{0}".format(item)] = 100*trySum["{0}".format(item)]/tryNum

 

        perM = max(acc, key=lambda i: acc[i])
        print(perM)
                    
        pygame.draw.rect(screen, [20,20,150], (50, 150, 30, 30))
    else:
        pygame.draw.rect(screen, [20,20,200], (50, 150, 30, 30))

 

    #Red
    if mouseClick and my > 150 and my < 280 and mx > 90 and mx < 120:
        pygame.draw.rect(screen, [255,255,255], (1, 1, box-1, box-1))
        pygame.draw.rect(screen, [150,20,20], (90, 150, 30, 30))
    else:
        pygame.draw.rect(screen, [200,20,20], (90, 150, 30, 30))

    pygame.display.flip()
    clock.tick(60)

 

pygame.quit()

Please reload

Read more:

iOSDC Japan 2018

September 7, 2018

Making an iOS Newsletter App with Updating Content using JSON and Swift 4

July 25, 2018

1/14
Please reload

    © 2019 Tech Kingdom

    Contact us : official.techkingdom@gmail.com