diff --git a/src/1.MachineLearning/Algorithm/gradientApprox.py b/src/1.MachineLearning/Algorithm/gradientApprox.py index a4a6054..f0d39f0 100644 --- a/src/1.MachineLearning/Algorithm/gradientApprox.py +++ b/src/1.MachineLearning/Algorithm/gradientApprox.py @@ -1,9 +1,109 @@ -from math import tan, tanh, exp, inf +#NPTEL - computation course + +#example1 +#1. check equal length +import numpy +import numpy as np + +def initMat(x): + """ + initializes a list x, transforming it into a numpy `array` + + input: + ----- + x : list vector [Warning: no bounded limit on its dimension is set ] + + + """ + return numpy.array(x) + + +#Theory +def dot1D(a , b): + + """returns the dot product + Assuming a , b are equal in dimensiom + """ + + return np.dot( a[i] * b[i]) + + + +def dot2D(a,b): + + """returns the dot product + Assuming a , b are equal in dimensiom + + Input: + ----- + a: vector, 1-Dimensional [ dim(a) = 1 ] + b: vector, 1-Dimensional [ dim(b) = 1 ] + + Processing: + ---------- + Note: assumes inputs: a,b are of the same dimension (size) + 1. each row in the first is multiplied with each column in the second vector b + Output: + ------- + + # Example + + >> + #Input: + ##1.inputs: + + a = [[1,2,3], [3,4,5]] + b = [[1,2,3], [3,4,5]] + ## 2. Processing: + >> + c = dot2D(a,b) + + # 3. Output: + >> print( c ) + >>[14, 50] + + """ + + l = [] + for i in range( len(a) ): + l.append( np.dot(a[i] , b[i] ) ) + return l + + +a = [[1,2,3], [3,4,5]] +b = [[1,2,3], [3,4,5]] + +A = [1,2,3] +B = [3,5,6] +c = dot2D(a,b) # DEbugs +print("c =",c) + +def init_vector(x): + return numpy.array(x) + +A = init_vector(A) +B = init_vector(B) +print("A = ", A) +print("B= ",B) +print("type(A) = ", type(A) ) +print("type(B)= ", type(B) ) + +A2 = A.T * A +print("A2 = ", A2) +I = numpy.invert(A).T * A +print("I = ", I) + +def LU(l,u): + return L * u + +def outerProduct(X,b): + return X @ b.T # or mat_mul() + class feedForwardNetwork: """ - Feed Forwad + Feed Forward by taking linear combination (of non-linearity ) inputs: ------ @@ -46,338 +146,169 @@ def __init__(self, w, x , phi, b): print("phi_w_i_j =",phi_w_i_j) -""" - recheck -======= - -""" -adamLutPhi -lutfipro91@gmail.com - -source: https://www.youtube.com/watch?v=jTzJ9zjC8nU&t=313s + +#outer product +A @ A.T - Loss function with weights episode : backprop - -Guide: +L = [1 , 0] +L = init_vector(L) +print(L) # [1 0] +#source : https://math.stackexchange.com/questions/4538838/how-to-prove-this-inverse-matrix-identity?rq=1 +a = [1, -1] +a = init_vector(a) -To differentiate +a_prime = numpy.invert(a) +I = a_prime * L * a +print("I= ", I) -We will use a neural network +#L * u -Having a neural Network, having an: +I = L.T * L +print(L) #the same vector [1 0] +print(I)# [ -2 -6 -12] +x = numpy.kron(L, L.T) +print("kron(L) = ",x) -activation function -activation function : +#source: https://www.youtube.com/watch?v=kTSKHXWlyl0 +dot_prod = 0.0 -having -layer1 = Layer( input ) --> layer2= Layer ( Hidden) +def isEqualDimension(a,b): + pass -still, need something in between, linking layer1 and layer2 +#example 4: -======= -1.we require a neural network -2.Having an activation function +A4= [ [1 ,- 6], [3,5]] # Rectangular (list) +# step1 transform data type ( list -to-> matrix) -layer1(input) , Layer2( input ) --> ... layern= Layer( Hidden)...ouput -map -- need something in between, linking layer1 and layer2 +##what if +A4_2= [ [1 - 6], [3,5]] #check output! +print("A4 = ",A4) +print("A4_2 = " , A4_2) +# assert A4 == A4_2 """ - -# class map - -class map : - def __init__(nnLayer1, nnLayer2): - - - nnlayer1. - +def get nCols(x): + >>> v = np.array([1,-1,1]) +>>> print(v[0:2]) +[1, -1] +>>> print(v[:]) +[ 1 -1 1] +>>> print(v[0:3:2]) +[1 1] +>>> C = np.array([[1,2,3],[4,5,6],[7,8,9]]) +>>> print(C) +[[1 2 3] + [4 5 6] + [7 8 9]] +>>> print(C[:,0]) +[1 4 7] """ -======= - pass - #nnlayer1. - - - -""" all derivatives of u with respect to the input x -Can be found +#ND Matrix -this is possible if I had 1 single neuron -what if: -I had 10 hidden Neurons(multiple hidden neurons) - -Even if multiple layers (of hidden neurons) through backpropagation) - -you can find out - -d output/ d input -du / dx - - - - -output/ d input -du / dx - - -#rank 2 approcimation -d2u / dx2 - -= -d2 u_hat/dx2 + linear_function - -d2 u_hat/dx2 + a* d_u_hat/dx - b ] - -this is the optimization algorithm -which we have to minimize -since it is a rank 2 -so - -forward propogation - -another episode - -w1 , w2 -> do forward prop +def initMatNd(x): + """ + The list comprehension method creates a new list of sublists, with each sublist corresponding to the number of rows in the matrix. We sliced sample_list to extract a chunk of “n_cols” elements from the original list, starting at index “i” and ending at index “i+n_cols”. -to get a new cost function -cross : ref Neural Network2: Cost function -credits L -https://arxiv.org/abs/2301.10581 + The range() function is used to generate a sequence of starting indices for each row, starting at 0 and incrementing by “n_cols”. -# straight line fit -Cscale = 0.3268 ˆσI + 0.0018. + However, it outputs a + list class. + That is because, technically, it is still a list, although it has been reshaped as a matrix. + """ + + print(type(x)) + #unviersity of Utah + n_cols = len(x[:]) # x[:,None,1] # len(x[:]) #, 0] # len( x[:,[0]]) + x = [x[i:i+n_cols] for i in range(0, len(x), n_cols)] + print(type(x) ) # it is still a list + +import numpy as np + +def dot2D(a,b): + """returns the dot product, for a 2D matrix + using numpy's einsum `eigenvalue sum -""" + -note : transport operator: while it does not make a copy, it is an in fix operator , has zero cost + + Assuming a , b are equal in dimensiom + note: a, b has to be on the same size , else it is undefined -class map: + [1 -6 + 3 5 ] . [1 5]T + """ - def __init__(self): - pass - -class nnLayer: - Nodes = [] - def __init__(self, vectorNodes): - """vector of nodes , by + return einsum(a, b) - Inputs: - ------- - vectorNodes: by default - """ - - _len = len(vectorNodes) - for i in range( _len +1 ): +def dotNd(k,m,a ): + i=0;j=0 + dot(a, b)[i,j,k,m] = sum(k[i,j,:] * b[k,:,m]) + return dot(a, b)[i,j,k,m] - self.Nodes.append(vectorNodes[i]) - - -class Node: +#DEMO +A4 = initMatNd(A4) +#I = dotNd(A4, np.transpose(A4) , 0) +#print("I", I) - def __init__(self, name, value): - self.name = name - self.value = value +#TODO: PEP-465 - def __init__2(self, name, x, w): - """ constructor : for scalar arguments - Fill the corresponding scalars to them - - input: +#In-fix operator - x: pattern - w: weight +#Application - """ - self.name = name - self.x = x - self.w = w +class Card: + """ - def __init__3(self, name, x, w): - """ - - If arguments are arrays, then fill a corresponding array with it - - """ - self.name = name - _len = len(x) - for i in range(_len+1): - self.X = x[i] - self.W = w[i] + source:https://stackoverflow.com/questions/932328/python-defining-my-own-operators + """ + + # other stuff + @staticmethod + def fromstring(s): + value, of, suit = s.split() + return Card(Value[value], Suit[suit]) + -#Gradient approximation +# for i in range( -#sum (L_t -= L) +#nptel : computation course ( taylor series ) + +def newtonRaphson(nSteps): + + """ 1. evaluate f, test its correct + 2. approximate solution L + F'(x ) nSteps = - F(x) + 3. next x= x_n+1 = x_n + lambda s -# w = w- a * dL/dw -> sum a Lt + steplength is imortant , so step s should b satisfactory + so, in a nonlinear solver: # use steph length control + local convergence: using liphschitz continuity + J calculation can be v inaccurate (causes problems ) + analytic J require some robostness + when difference Jacobian is done poorly -def funTan(x): - return tan(x) + 1 quanitgy stagnation (how) + 1. add errrors in function evaluagio, & deruvatuve eval tio theorem 1.1 -def _2Diff(x): - """source: Corollary 2: """ - - return tan( tan( x) ) + 1 -""" - assert funTanh(1) == funTanh2(1) #passed - assert funTanh(1) != funTan(1) """ + return fac(nSteps) -def sigmoid(x): - """ - Sigmoid function: - source(wikipedia): - https://en.wikipedia.org/wiki/Sigmoid_function - - Source: https://math.stackexchange.com/questions/1226089/is-expx-the-same-as-ex - mathematicians tend to use exp instead of e function for `interesting` calculations - - """ - return 1 / (1+ exp(x)) - - -# user is free to pick an activation function -def activationFun(x): - - sigmoid(x) - -def getOutput(inputNodes): - """returns the output of input nodes - inputs: - ------- - inputNodes: a list of nodes - - processing: - ---------- - Algorithm: - 1. get `inputNodes` length: - - nodeLength = len(Nodes) - - output: - ------- - The sum of all input nodes (assuming Neural nodes are `fully-connected` - - """ - - -#activationFun(x) - -def getNode(x, neuralOutput): - """ me: observe: - Any node is: - 1. fully-connected: has got to be connected with - the `Output` of - Neural Nodes (From the previous layer) - - - """ - return activationFun(x, neuralOutput) - -def g(x): - """ the gradient of x - - Reasoning - - Assume : - 1. u is a neural network , where x is an input - me: with tan(x) as an ectivation function - _sum = 0 - for i in range ( n +1) - _sum += Node(i) - - me - where Node is a weighted - sigmoid(w1, x) = sigmoid ( w1*x ) - - - universal appproximation theorem - - I can always approxumatge you , arbitrary close - to the neural network - - I can assume u is a neural network - how does that help us ? - A. - example: - 10 nn input nodes - with 1 output - - writing - a full Functional Form for u - - forget the bias unit - - - say: - Hidden Layer : - - - 1 hidden neuron node (a) - - where - a1 = sigmoid ( w , x) - suppose this layer is linear - u = w2 * sigmoid( w1 * x ) - analytical ingestigation: - - du / d sigmoid = w2 . sigmoid' * (w1 * x) * w - - """ - - return tan(x) - -def H(x): - return g (g(x)) - -def G(x,n): - """A generalized Gradient """ - if n==1: - return g(x) - else: - n = n-1 - return G(x,n) - -#Norm -def norm(x): - - #placeholder : TODO: replace with the actual norm function - #if x is scalar: return its absolute value - - return abs(x) - -#Global (TODO) -Lambda = 2 # The Regularization parameter - - - -def h(t=1): - """TODO: plug in an actual characteristic function """ - return t * 1 - -def checkLambda(Lambda): - - if Lambda > 1: - pass +def LevenbergMarquardt(x): + """an update on the NewtonRaphson Method """ pass -#needs testing to pass -def taylor(x): - """ gets taylor series, 2nd order """ - return ( f(x) + (x - x0) * g(x) ) + 1/2 * ((x-x0) * H(x)) - -def getH( n, t=1, Lambda=2): - """solves next h_t+n """ - - return norm(h(t)) * Lambda **n - - -print("h(1) = ", getH( n = 1) ) - -print("h(2) = ", getH( n = 2 ) ) - +import sklearn +#uses: threadpoolctl +def getKmeeans(nClusters = 5, maxIterations =500, eps_tolerance= 1.00e-4): + return sklearn.cluster.KMeans(n_clusters=nClusters, init='k-means++', n_init='warn', max_iter=maxIterations, tol=eps_tolerance, verbose=0, random_state=None, copy_x=True, algorithm= 'svd') #'lloyd' +