Difference between Ordinary Least Square (OLS) and Gradient Descent to find best fit line

Ordinary Least Square(OLS):

- Non Iterative method to find best fit line such that the sum of squares of diff of Observed and Predicted values is minimized.

Error = (y_pred – y_act)^2
Line => y = bo + b1x

y_i =  Actual Value

- Above formula is for Univariate(one variable)

- For multivariate case, when we have many variables, the formula becomes complicated and requires too much calculation while implementing in software. 

- fail for collinear predictors(correlation between features)

- can be run in parallel but its still much complicated and expensive.

Gradient Descent:

- finds the linear model parameters iteratively.

- applies to non-linear model as well.

- works well for collinear predictors

- saves lot of time in calculation as it can be run parallely and distribute load across multiple processors.

•Cost Function, J(m,c) = (y_pred – y_act)^2 / No. of data point
•Hypothesis: y_pred = c + mx

Deep Learning: Create Neural Network to Recognize plus and cross image

Problem:
1> Consider Image of size 3*3 for plus and cross
2> In terms of input plus = [0,1,0,1,1,1,0,1,0], cross = [1,0,1,0,1,0,1,0,1]


Algorithm:
1> Error at Output Layer = (Out_exp - Oct_pre)*Oct_pre*(1 - Oct_pre)
2>  Re calculate Weight = Weight + learning*error*input
3> Error at Hidden Layer = w11*error1 + w12*error2

Code:
import numpy as np
import os
import random
import cv2, os
import matplotlib.pyplot as plt
import math

lr = 1 #learning rate
#Initialize Bias and Weights
bias = [random.random() for x in range(4)]
bias_out = [random.random() for x in range(2)]
h, w = 10, 5
weight_hid = [[random.random() for x in range(w)] for y in range(h)]
h, w = 5, 3
weight_out = [[random.random() for x in range(w)] for y in range(h)]

def flatten_image(image_path):
    image = cv2.imread(image_path)
    image_grayscale = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY )
    size=(3,3)
    resized_image = cv2.resize(image_grayscale, size)
    image_flatten = resized_image.flatten()
    for i in range(len(image_flatten)):
        if image_flatten[i] > 150:
            image_flatten[i] = 1
        else:
            image_flatten[i] = 0
    return image_flatten    

def sigmoid(x):
    return 1/(1+math.exp(-x))

def hidden_layer(x):
    hid_outputP = [0,0,0,0]
    #hid_error = [0,0,0,0]

    for i in range(0,4):
        hid_outputP[i] = bias[i]*weight_hid[0][i+1]
        for j in range(0,9):
            hid_outputP[i] =  hid_outputP[i] + x[j]*weight_hid[j+1][i+1]
         
        hid_outputP[i] = sigmoid(hid_outputP[i])
    return hid_outputP

def output_layer(h, result, plus):
    outputP = [0,0]
    error = [0,0]
    for i in range(0,2):
        outputP[i] = bias_out[i]
        for j in range(0,4):
            outputP[i] =  outputP[i] + h[j]*weight_out[j+1][i+1]

    outputP[0] = sigmoid(outputP[0])
    outputP[1] = sigmoid(outputP[1])
    #Calculate Error at Output Layer
    error[0] = (result[0] - outputP[0])*outputP[0]*(1 - outputP[0])
    error[1] = (result[1] - outputP[1])*outputP[1]*(1 - outputP[1])

    #Calculate Error at Hidden Layer
    for i in range(1,5):
        error_hid[i] = (error[0]*weight_out[i][1] + error[1]*weight_out[i][2])*h[i-1]*(1 - h[i-1])
    #Recalculate Weight to Output Layer
    for i in range(1,5):
        weight_out[i][1] =  weight_out[i][1] + error[0]*h[i-1]*lr
        weight_out[i][2] =  weight_out[i][2] + error[1]*h[i-1]*lr
       
    #Recalculate Weight to Hidden Layer
    for i in range(1,5):
        for j in range(1,10):
            weight_hid[j][i] = weight_hid[j][i] + error_hid[i]*plus[j-1]

def predict(input):
    input_hid = hidden_layer(input)
    outputP = [0,0]
    for i in range(0,2):
        outputP[i] = bias_out[i]
        for j in range(0,4):
            outputP[i] =  outputP[i] + input_hid[j]*weight_out[j+1][i+1]

    outputP[0] = sigmoid(outputP[0])
    outputP[1] = sigmoid(outputP[1])   
    print(outputP[0],outputP[1]) 
 
result_plus = [0.9, 0.1]
result_cross = [0.1, 0.9]
error_hid = [0,0,0,0,0]

#Image can be taken from : https://github.com/bansalrishi/MachineLearningWithPython_UD

#E.g: Path in GitHub: Data/NN/plus.jpg

plus_path = 'Data/NN/plus.jpg'
plus = flatten_image(plus_path)
plus_cross = 'Data/NN/cross.jpg'
cross = flatten_image(plus_cross)

for i in range(200) :
    #print("Running loop i=%s"%i)   
    H = hidden_layer(plus)
    output_layer(H, result_plus, plus)
    H = hidden_layer(cross)
    output_layer(H, result_cross,cross)
   
print("Giving input as cross")
predict(cross)
print("Giving input as plus")
predict(plus)


Output:

Giving input as cross
0.12235658465850822 0.8819751642398935
Giving input as plus
0.881445602118824 0.11705488953094145