From ce9995f41aaa5d179b9caddd3607d8aee8fb300b Mon Sep 17 00:00:00 2001 From: Ahmad Lutfi Date: Fri, 14 Jul 2023 00:22:42 -0400 Subject: [PATCH 1/2] Update gradientApprox.py --- .../Algorithm/gradientApprox.py | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/src/1.MachineLearning/Algorithm/gradientApprox.py b/src/1.MachineLearning/Algorithm/gradientApprox.py index d25ccbd..dcfb234 100644 --- a/src/1.MachineLearning/Algorithm/gradientApprox.py +++ b/src/1.MachineLearning/Algorithm/gradientApprox.py @@ -1,35 +1,28 @@ from math import tan, tanh, exp, inf """ - recheck +source: https://www.youtube.com/watch?v=jTzJ9zjC8nU&t=313s + Loss function with weights episode : backprop Guide: To differentiate -We will use a neural network - -Having a neural Network, having an: - -activation function - - -activation function : - -having -layer1 = Layer( input ) --> layer2= Layer ( Hidden) - -still, need something in between, linking layer1 and layer2 +1.we require a neural network +2.Having an activation function +layer1(input) , Layer2( input ) --> ... layern= Layer( Hidden)...ouput +map +- need something in between, linking layer1 and layer2 +""" # class map class map : def __init__(nnLayer1, nnLayer2): + pass + #nnlayer1. - nnlayer1. - -""" """ all derivatives of u with respect to the input x Can be found @@ -41,11 +34,10 @@ def __init__(nnLayer1, nnLayer2): Even if multiple layers (of hidden neurons) through backpropagation) you can find out -d output/ d input +output/ d input du / dx - #rank 2 approcimation d2u / dx2 From 11f37fcfadcfb9b5b8d818a887ed08321ab13350 Mon Sep 17 00:00:00 2001 From: Ahmad Lutfi Date: Fri, 14 Jul 2023 00:47:15 -0400 Subject: [PATCH 2/2] Update gradientApprox.py --- .../Algorithm/gradientApprox.py | 517 +++++++++--------- 1 file changed, 266 insertions(+), 251 deletions(-) diff --git a/src/1.MachineLearning/Algorithm/gradientApprox.py b/src/1.MachineLearning/Algorithm/gradientApprox.py index dcfb234..f0d39f0 100644 --- a/src/1.MachineLearning/Algorithm/gradientApprox.py +++ b/src/1.MachineLearning/Algorithm/gradientApprox.py @@ -1,299 +1,314 @@ -from math import tan, tanh, exp, inf +#NPTEL - computation course -""" -source: https://www.youtube.com/watch?v=jTzJ9zjC8nU&t=313s - - Loss function with weights episode : backprop - -Guide: - -To differentiate - -1.we require a neural network -2.Having an activation function - -layer1(input) , Layer2( input ) --> ... layern= Layer( Hidden)...ouput -map -- need something in between, linking layer1 and layer2 -""" -# class map - -class map : - def __init__(nnLayer1, nnLayer2): - pass - #nnlayer1. - - -""" all derivatives of u with respect to the input x -Can be found - -this is possible if I had 1 single neuron -what if: -I had 10 hidden Neurons(multiple hidden neurons) - -Even if multiple layers (of hidden neurons) through backpropagation) - -you can find out -output/ d input -du / dx - - -#rank 2 approcimation -d2u / dx2 +#example1 +#1. check equal length +import numpy +import numpy as np -= -d2 u_hat/dx2 + linear_function +def initMat(x): + """ + initializes a list x, transforming it into a numpy `array` -d2 u_hat/dx2 + a* d_u_hat/dx - b ] + input: + ----- + x : list vector [Warning: no bounded limit on its dimension is set ] -this is the optimization algorithm -which we have to minimize -since it is a rank 2 -so + + """ + return numpy.array(x) -forward propogation -another episode +#Theory +def dot1D(a , b): -w1 , w2 -> do forward prop + """returns the dot product + Assuming a , b are equal in dimensiom + """ + + return np.dot( a[i] * b[i]) + -to get a new cost function -cross : ref Neural Network2: Cost function -credits L -https://arxiv.org/abs/2301.10581 -# straight line fit -Cscale = 0.3268 ˆσI + 0.0018. +def dot2D(a,b): + """returns the dot product + Assuming a , b are equal in dimensiom -""" - - -class map: - - def __init__(self): - pass - -class nnLayer: - Nodes = [] - def __init__(self, vectorNodes): - """vector of nodes , by - - Inputs: - ------- - vectorNodes: by default - - """ - - _len = len(vectorNodes) - for i in range( _len +1 ): - - self.Nodes.append(vectorNodes[i]) - - -class Node: - - def __init__(self, name, value): - - self.name = name - self.value = value - - def __init__2(self, name, x, w): - """ constructor : for scalar arguments - Fill the corresponding scalars to them - - input: - - x: pattern - w: weight - - """ - self.name = name - self.x = x - self.w = w - - def __init__3(self, name, x, w): - """ - - If arguments are arrays, then fill a corresponding array with it - - """ - self.name = name - _len = len(x) - for i in range(_len+1): - self.X = x[i] - self.W = w[i] + Input: + ----- + a: vector, 1-Dimensional [ dim(a) = 1 ] + b: vector, 1-Dimensional [ dim(b) = 1 ] + Processing: + ---------- + Note: assumes inputs: a,b are of the same dimension (size) + 1. each row in the first is multiplied with each column in the second vector b + Output: + ------- + + # Example + + >> + #Input: + ##1.inputs: + + a = [[1,2,3], [3,4,5]] + b = [[1,2,3], [3,4,5]] + ## 2. Processing: + >> + c = dot2D(a,b) + + # 3. Output: + >> print( c ) + >>[14, 50] + + """ + + l = [] + for i in range( len(a) ): + l.append( np.dot(a[i] , b[i] ) ) + return l + + +a = [[1,2,3], [3,4,5]] +b = [[1,2,3], [3,4,5]] + +A = [1,2,3] +B = [3,5,6] +c = dot2D(a,b) # DEbugs +print("c =",c) + +def init_vector(x): + return numpy.array(x) + +A = init_vector(A) +B = init_vector(B) +print("A = ", A) +print("B= ",B) +print("type(A) = ", type(A) ) +print("type(B)= ", type(B) ) + +A2 = A.T * A +print("A2 = ", A2) +I = numpy.invert(A).T * A +print("I = ", I) + +def LU(l,u): + return L * u + +def outerProduct(X,b): + return X @ b.T # or mat_mul() + + +class feedForwardNetwork: + + """ + Feed Forward + by taking linear combination (of non-linearity ) + inputs: + ------ + w: weights vector + x : values vector + """ + + def __init__(self, w, x , phi, b): + + self.bias = bias + + _len = len(w) + + for i in range(_len+1): + + + #Calc others: + self.w[i] = w + self.x[i] = x + + + self.phi_w = [] # list of phi , of w , at + + phi_w_i_j = 0.0 + + #bias + for j in range(len(bias) ): + + #insert scalar bias b into list entry j + self.bias[j] = b + #activation (characteristic function) phi + self.phi[j] = phi + + # calculate the weighted average + w_i_j += w * x # wieghted average + + self.phi_w[ j ] = phi(w_i_j) + + phi_w_i_j += self.phi_w[ j ] + b # dim [bias = N*M + b ] + print("phi_w_i_j =",phi_w_i_j) + + + +#outer product +A @ A.T + + +L = [1 , 0] +L = init_vector(L) +print(L) # [1 0] +#source : https://math.stackexchange.com/questions/4538838/how-to-prove-this-inverse-matrix-identity?rq=1 +a = [1, -1] +a = init_vector(a) + +a_prime = numpy.invert(a) +I = a_prime * L * a +print("I= ", I) + +#L * u + +I = L.T * L +print(L) #the same vector [1 0] +print(I)# [ -2 -6 -12] +x = numpy.kron(L, L.T) +print("kron(L) = ",x) -#Gradient approximation -#sum (L_t -= L) -# w = w- a * dL/dw -> sum a Lt +#source: https://www.youtube.com/watch?v=kTSKHXWlyl0 +dot_prod = 0.0 +def isEqualDimension(a,b): + pass +#example 4: -def funTan(x): - return tan(x) +A4= [ [1 ,- 6], [3,5]] # Rectangular (list) +# step1 transform data type ( list -to-> matrix) -def _2Diff(x): - """source: Corollary 2: """ - - return tan( tan( x) ) + 1 +##what if +A4_2= [ [1 - 6], [3,5]] #check output! +print("A4 = ",A4) +print("A4_2 = " , A4_2) +# assert A4 == A4_2 """ - assert funTanh(1) == funTanh2(1) #passed - - assert funTanh(1) != funTan(1) +def get nCols(x): + >>> v = np.array([1,-1,1]) +>>> print(v[0:2]) +[1, -1] +>>> print(v[:]) +[ 1 -1 1] +>>> print(v[0:3:2]) +[1 1] +>>> C = np.array([[1,2,3],[4,5,6],[7,8,9]]) +>>> print(C) +[[1 2 3] + [4 5 6] + [7 8 9]] +>>> print(C[:,0]) +[1 4 7] """ +#ND Matrix + +def initMatNd(x): + """ + The list comprehension method creates a new list of sublists, with each sublist corresponding to the number of rows in the matrix. We sliced sample_list to extract a chunk of “n_cols” elements from the original list, starting at index “i” and ending at index “i+n_cols”. + + The range() function is used to generate a sequence of starting indices for each row, starting at 0 and incrementing by “n_cols”. + + However, it outputs a + list class. + That is because, technically, it is still a list, although it has been reshaped as a matrix. + """ + + print(type(x)) + #unviersity of Utah + n_cols = len(x[:]) # x[:,None,1] # len(x[:]) #, 0] # len( x[:,[0]]) + x = [x[i:i+n_cols] for i in range(0, len(x), n_cols)] + print(type(x) ) # it is still a list + +import numpy as np + +def dot2D(a,b): -def sigmoid(x): - """ - Sigmoid function: - source(wikipedia): - https://en.wikipedia.org/wiki/Sigmoid_function + """returns the dot product, for a 2D matrix + using numpy's einsum `eigenvalue sum - Source: https://math.stackexchange.com/questions/1226089/is-expx-the-same-as-ex - mathematicians tend to use exp instead of e function for `interesting` calculations + -note : transport operator: while it does not make a copy, it is an in fix operator , has zero cost - """ - return 1 / (1+ exp(x)) - - -# user is free to pick an activation function -def activationFun(x): + + Assuming a , b are equal in dimensiom + note: a, b has to be on the same size , else it is undefined - sigmoid(x) + [1 -6 + 3 5 ] . [1 5]T + """ -def getOutput(inputNodes): - """returns the output of input nodes - inputs: - ------- - inputNodes: a list of nodes + return einsum(a, b) - processing: - ---------- - Algorithm: - 1. get `inputNodes` length: - - nodeLength = len(Nodes) - - output: - ------- - The sum of all input nodes (assuming Neural nodes are `fully-connected` - """ +def dotNd(k,m,a ): + i=0;j=0 + dot(a, b)[i,j,k,m] = sum(k[i,j,:] * b[k,:,m]) + return dot(a, b)[i,j,k,m] - -#activationFun(x) - -def getNode(x, neuralOutput): - """ me: observe: - Any node is: - 1. fully-connected: has got to be connected with - the `Output` of - Neural Nodes (From the previous layer) +#DEMO +A4 = initMatNd(A4) +#I = dotNd(A4, np.transpose(A4) , 0) +#print("I", I) - """ - return activationFun(x, neuralOutput) +#TODO: PEP-465 -def g(x): - """ the gradient of x - - Reasoning - - Assume : - 1. u is a neural network , where x is an input - me: with tan(x) as an ectivation function - _sum = 0 - for i in range ( n +1) - _sum += Node(i) +#In-fix operator - me - where Node is a weighted - sigmoid(w1, x) = sigmoid ( w1*x ) +#Application +class Card: + """ - universal appproximation theorem + source:https://stackoverflow.com/questions/932328/python-defining-my-own-operators - I can always approxumatge you , arbitrary close - to the neural network + """ + + # other stuff + @staticmethod + def fromstring(s): + value, of, suit = s.split() + return Card(Value[value], Suit[suit]) + - I can assume u is a neural network - how does that help us ? - A. - example: - 10 nn input nodes - with 1 output +# for i in range( - writing - a full Functional Form for u +#nptel : computation course ( taylor series ) + +def newtonRaphson(nSteps): + + """ 1. evaluate f, test its correct + 2. approximate solution L + F'(x ) nSteps = - F(x) + 3. next x= x_n+1 = x_n + lambda s - forget the bias unit + steplength is imortant , so step s should b satisfactory + so, in a nonlinear solver: # use steph length control - - say: - Hidden Layer : - - 1 hidden neuron node (a) + local convergence: using liphschitz continuity + J calculation can be v inaccurate (causes problems ) + analytic J require some robostness + when difference Jacobian is done poorly - where - a1 = sigmoid ( w , x) - suppose this layer is linear - u = w2 * sigmoid( w1 * x ) - analytical ingestigation: - - du / d sigmoid = w2 . sigmoid' * (w1 * x) * w - - """ - - return tan(x) + 1 quanitgy stagnation (how) + 1. add errrors in function evaluagio, & deruvatuve eval tio theorem 1.1 -def H(x): - return g (g(x)) -def G(x,n): - """A generalized Gradient """ - if n==1: - return g(x) - else: - n = n-1 - return G(x,n) - -#Norm -def norm(x): - - #placeholder : TODO: replace with the actual norm function - #if x is scalar: return its absolute value - - return abs(x) - -#Global (TODO) -Lambda = 2 # The Regularization parameter - - - -def h(t=1): - """TODO: plug in an actual characteristic function """ - return t * 1 - -def checkLambda(Lambda): +""" + return fac(nSteps) - if Lambda > 1: - pass +def LevenbergMarquardt(x): + """an update on the NewtonRaphson Method """ pass -#needs testing to pass -def taylor(x): - """ gets taylor series, 2nd order """ - return ( f(x) + (x - x0) * g(x) ) + 1/2 * ((x-x0) * H(x)) - -def getH( n, t=1, Lambda=2): - """solves next h_t+n """ - - return norm(h(t)) * Lambda **n - - -print("h(1) = ", getH( n = 1) ) - -print("h(2) = ", getH( n = 2 ) ) - +import sklearn +#uses: threadpoolctl +def getKmeeans(nClusters = 5, maxIterations =500, eps_tolerance= 1.00e-4): + return sklearn.cluster.KMeans(n_clusters=nClusters, init='k-means++', n_init='warn', max_iter=maxIterations, tol=eps_tolerance, verbose=0, random_state=None, copy_x=True, algorithm= 'svd') #'lloyd' +