[ { "path": "Adaboost/README.md", "content": "Adaboost 算法实现。\n\n更多机器学习深度学习博客请关注CSDN博客：[LiuLongpo](http://blog.csdn.net/llp1992)\n" }, { "path": "Adaboost/adaboost.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Thu Jun 11 12:42:48 2015\n\n@author: liu\n\"\"\"\nfrom numpy import *\n\ndef loadSimpData():\n\n dataMat = array([[1.,2.1],[1.5,1.6],[1.3,1.],[1.,1.],[2.,1.],[1.2,1.1],\\\n [1.1,0.4],[0.9,1.3],[0.86,1.2],[1.8,1.8],[1.7,1.5],[1.9,1.8]])\n classLabels = array([1.0,1.0,-1.0,-1.0,1.0,-1.0,-1.0,-1.0,\\\n -1.0,1.0,1.0,1.0])\n return dataMat,classLabels\n\n# 单树桩分类器，也就是简单的单层决策树弱分类器\n# 该函数根据某个最好的特征的最好划分点对数据进行分类\n# demen就是特征，threshVal就是划分点，retArray就是返回的分类结果\ndef stumpClassify(dataMatrix,dimen,threshVal,threshIneq):\n retArray = ones((shape(dataMatrix)[0],1))\n if threshIneq == 'lt':\n retArray[dataMatrix[:,dimen]<=threshVal] = -1.0\n else:\n retArray[dataMatrix[:,dimen]>threshVal] = -1.0\n return retArray\n \n# 创建树桩，返回最好的树桩和该树桩的分类最小误差以及分类结果\n# 该函数用于从数据集中找到最好的划分特征以及该特征的最好划分点\ndef buildStump(dataArr,classLabels,D):\n # 建立备份的数据\n dataMatrix = mat(dataArr);\n labelMat = mat(classLabels).T\n # m 是行，也就是每个样本，n是列，也就是每个特征\n m,n = shape(dataMatrix)\n # 步数的设置，也就是在每个特征的最大值和最小值中分几次设置阈值进行数据划分\n # 也就是获取这个特征的最佳划分点，步数越高，特征点寻找得越精细，但耗时更多\n # 用字典来存储bestStump的数据\n numSteps = 10.0;bestStump = {};bestClassEst = mat(zeros((m,1)))\n minError = inf\n # 对每个特征\n for i in range(n):\n # 获取每个特征的最小值和最大值\n rangeMin = dataMatrix[:,i].min();\n rangeMax = dataMatrix[:,i].max();\n stepSize = (rangeMax-rangeMin)/numSteps\n for j in range(-1,int(numSteps)+1):\n for inequal in ['lt','gt']:\n threshVal = (rangeMin + float(j) * stepSize)\n # 预测值\n predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal)\n errArr = mat(ones((m,1)))\n errArr[predictedVals == labelMat] = 0\n # 样本权重乘以样本误差\n weightedError = D.T * errArr\n # 下面的 .2f 表示浮点数小数殿后两位， .3f 表示小数点后3位\n print \"split:dim %d,thresh %.2f, thresh ineqal: %s,the weighted eror is %.3f\" %\\\n (i,threshVal,inequal,weightedError)\n if weightedError < minError:\n minError = weightedError\n # 最好的分类结果\n bestClassEst = predictedVals.copy()\n bestStump['dim'] = i\n bestStump['thresh'] = threshVal\n bestStump['ineq'] = inequal\n return bestStump,minError,bestClassEst\n \ndef adaBoostTrainDS(dataArr,classLabels,numIt = 40):\n weakClassArr = []\n m = shape(dataArr)[0]\n D = mat(ones((m,1))/m)\n # 创建矩阵 mat\n aggClassEst = mat(zeros((m,1)))\n for i in range(numIt):\n bestStump,error,classEst = buildStump(dataArr,classLabels,D)\n print 'D:',D.T\n # log 就是 ln 公式： a = 0.5*ln((1-e)/e)\n alpha = float(0.5*log((1.0-error)/max(error,1e-16)))\n bestStump['alpha'] = alpha\n # 将当前的最好的树桩添加到弱分类其数组中\n weakClassArr.append(bestStump)\n print 'classEst:',classEst.T\n # D权重的更新公式，利用原本的类别classLabels与划分的类别classEst做乘积\n # 用来同时计算正确划分和错误划分的公式，也就是自动确定正负号\n expon = multiply(-1*alpha*mat(classLabels).T,classEst)\n # 更新权重D\n D = multiply(D,exp(expon))\n D = D/D.sum()\n aggClassEst += alpha*classEst\n print 'aggClassEst:',aggClassEst.T\n aggErrors = multiply(sign(aggClassEst)!=mat(classLabels).T,ones((m,1)))\n errorRate = aggErrors.sum()/m\n print 'total error:',errorRate,\"\\n\"\n if errorRate == 0.0:break;\n return weakClassArr\n \n # 利用学习得到的多个级联弱分类器进行数据分类\ndef adaClassify(datToClass,classifierArr):\n dataMatrix = mat(datToClass)\n m = shape(dataMatrix)[0]\n aggClassEst = mat(zeros((m,1)))\n for i in range(len(classifierArr)):\n classEst = stumpClassify(dataMatrix,classifierArr[i]['dim'],\\\n classifierArr[i]['thresh'],classifierArr[i]['ineq'])\n aggClassEst += classifierArr[i]['alpha'] * classEst\n print aggClassEst\n return sign(aggClassEst)\n" }, { "path": "Adaboost/testAdaboost.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Thu Jun 11 12:57:27 2015\n\n@author: LiuLongpo\n\"\"\"\nimport optunity\nimport adaboost\nimport matplotlib.pyplot as plt\nfrom numpy import *\ndataMat,classLabels = adaboost.loadSimpData()\n\n#plt.scatter(dataMat[:,0],dataMat[:,1])\n# D是样本的权重矩阵\nD = mat(ones((5,1))/5)\n#adaboost.buildStump(dataMat,classLabels,D)\nprint 'data train...'\nclassifierArr = adaboost.adaBoostTrainDS(dataMat,classLabels,30)\nprint 'getClassifier:',classifierArr\nprint 'data predict...'\n# 学习得到3个分类器，predict时，每一个分类器级联分类得到的预测累加值 \n# aggClassEst越来越远离0，也就是正越大或负越大，也就是分类结果越来越强\nadaboost.adaClassify([[1,0.8],[1.8,2]],classifierArr)\n# 0,lt,1.3 1,lt,1.0 0,lt,0.9\nplt.figure()\nI = nonzero(classLabels>0)[0]\nplt.scatter(dataMat[I,0],dataMat[I,1],s=60,c=u'r',marker=u'o')\nI = nonzero(classLabels<0)[0]\nplt.scatter(dataMat[I,0],dataMat[I,1],s=60,c=u'b',marker=u'o')\n\n\nplt.plot([1.32,1.32],[0.5,2.5])\nplt.plot([0.5,2.5],[1.42,1.42])\nplt.plot([0.97,0.97],[0.5,2.5])\n\n'''\nplt.figure()\nI = nonzero(classLabels>0)[0]\nplt.scatter(dataMat[I,0],dataMat[I,1],s=60,c=u'r',marker=u'o')\nI = nonzero(classLabels<0)[0]\nplt.scatter(dataMat[I,0],dataMat[I,1],s=60,c=u'b',marker=u'o')\nplt.plot([1.32*1.19,1.32*1.19],[0.5,2.5])\nplt.plot([0.5,2.5],[1.42*1.52,1.42*1.52])\nplt.plot([0.97*1.13,0.97*1.13],[0.5,2.5])\n#plt.scatter([0,5],[0,5])\n\n'''\n" }, { "path": "Decision-Tree/README.md", "content": "决策树算法实现。\n\n更多机器学习深度学习博客请关注CSDN博客：[LiuLongpo](http://blog.csdn.net/llp1992)\n" }, { "path": "Decision-Tree/TestTree.py", "content": "import Tree\n\ndataSet,label = Tree.createDataSet()\nprint 'dataSet:' , dataSet\n#Tree.createTree(dataSet,label)\n" }, { "path": "Decision-Tree/Tree.py", "content": "from math import log\n# 计算熵\ndef calcShannonEnt(dataSet):\n numEntries = len(dataSet)\n labelCounts = {}\n for featVec in dataSet:\n currentLabel = featVec[-1]\n if currentLabel not in labelCounts.keys():\n labelCounts[currentLabel] = 0\n labelCounts[currentLabel] += 1\n shannoEnt = 0.0\n for key in labelCounts:\n prob = float(labelCounts[key])/numEntries\n shannoEnt -= prob * log(prob,2)\n return shannoEnt\ndef createDataSet():\n dataSet = [[1,1,0,'yes'],[1,1,1,'yes'],[1,0,1,'no'],[0,1,0,'no'],[0,1,0,'no']]\n label = ['no surfacing','flippers']\n return dataSet,label\n# 划分数据集\n# axis为特征，也就是对某一个特征进行判定，比如身高\n# value为特征的值，也就是说，某一个特征的不同值，\n# 比如身高这个特征有高和矮之分，高跟矮就是这个value\n# 数据集为 [ [1,1,0,'yes'],[1,1,1,'yes'],[1,0,1,'no'],[0,1,0,'no'],[0,1,0,'no'] ]\n# 每一个 feaVec 就是一行，也就是一个样本数据\ndef spiltDataSet(dataSet,axis,value):\n retDataSet = []\n for featVec in dataSet:\n # 如果当前样本数据的特征 axis 的值 featVec[axis] 与我们要求的value相等\n # 也就是我们认为当前的样本符合我们的要求\n if featVec[axis] == value:\n # 对于符合要求的样本数据\n # 我们将这个特征剪掉，剩下的数据组成一个新的子样本并返回\n redecedFeatVec = featVec[:axis]\n redecedFeatVec.extend(featVec[axis+1:])\n retDataSet.append(redecedFeatVec)\n return retDataSet\n# 寻找最优的划分特征\ndef chooseBestFeatureToSplit(dataSet):\n # 为什么减一？\n numFeatures = len(dataSet[0])-1\n # 计算源数据的熵\n baseEntropy = calcShannonEnt(dataSet)\n bestInfoGain = 0.0\n bestFeature = -1\n for i in xrange(numFeatures):\n # 获取每一列，因为每一列都属于一个特征的不同value\n featList = [example[i] for example in dataSet]\n # 确保每个值都是唯一的\n uniqueVals = set(featList)\n newEntorpy = 0.0\n print 'spilt feature ', i\n # 对每个value进行划分\n for value in uniqueVals:\n subDataSet = spiltDataSet(dataSet,i,value)\n print 'subDataSet:' , subDataSet\n prob = len(subDataSet)/float(len(dataSet))\n # 计算该特征进行划分后的信息\n newEntorpy += prob * calcShannonEnt(subDataSet)\n # 计算该特征进行划分后的信息增益\n infoGain = baseEntropy - newEntorpy\n # 寻找信息增益最大的特征就是最好划分的特征\n if (infoGain > bestInfoGain):\n bestInfoGain = infoGain\n bestFeature = i\n print 'BestFeatureToSplit' \n print bestFeature\n return bestFeature\n# 投票表决，获取概率最大的那个分类\ndef majorityCnt(classList):\n classCount = {}\n for vote in classList:\n if vote not in classCount.keys():classCount[vote] = 0\n classCount[vote] += 1\n# 反向排序，也就是从大到小\n sortedClassCount = sorted(classCount.iteritems(),key = operator.itemgetter(1),reverse=True)\n return sortedClassCount[0][0]\ndef createTree(dataSet,labels):\n # 获取每个样本数据的分类\n classList = [example[-1] for example in dataSet]\n # 第一个迭代终止条件：选取第一个classList中的类，判断它出现的次数是否与\n # classList的长度相等，如果相等，证明classList中的所有类已经被分为同一种类\n # 则返回该类\n if classList.count(classList[0]) == len(classList):\n return classList[0]\n #如果所有特征都迭代完了，此时还没返回，\n # 说明当前仍然不能讲数据集划分为仅包含唯一类别的分组\n # 此时则返回次数出现最多的那个类别最为当前数据集的类别\n if len(dataSet[0]) == 1:\n return majorityCnt(classList)\n # 寻找最优的划分特征\n bestFeat = chooseBestFeatureToSplit(dataSet)\n bestFeatLabel = labels[bestFeat]\n myTree = {bestFeatLabel:{}}\n del(labels[bestFeat])\n # 获取最优特征的各个属性\n featValues = [example[bestFeat] for example in dataSet] \n uniqueVals = set(featValues)\n for value in uniqueVals:\n subLabels = labels[:]\n # 迭代创建决策树\n myTree[bestFeatLabel][value] = createTree(spiltDataSet(dataSet,bestFeat,value),subLabels)\n return myTree\n" }, { "path": "DeepLearning/CNN_cifar-10/cifar.py", "content": "\n# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Thu Aug 27 11:27:34 2015\n\n@author: lab-liu.longpo\n\"\"\"\n\n\nfrom __future__ import absolute_import\nfrom __future__ import print_function\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D\nfrom keras.optimizers import SGD, Adadelta, Adagrad\nfrom keras.utils import np_utils, generic_utils\n\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport scipy.io as sio\n\nd = sio.loadmat('data.mat')\ndata = d['d']\nlabel = d['l']\ndata = np.reshape(data,(50000,3,32,32))\nlabel = np_utils.to_categorical(label, 10)\n\nprint ('finish loading data')\n\nmodel = Sequential()\n\nmodel.add(Convolution2D(32, 3, 5, 5, border_mode='valid')) \nmodel.add(Activation('relu'))\n#model.add(MaxPooling2D(poolsize=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Convolution2D(32, 32, 5, 5, border_mode='valid')) \nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(poolsize=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Convolution2D(64, 32, 3, 3, border_mode='valid')) \nmodel.add(Activation('relu'))\nmodel.add(MaxPooling2D(poolsize=(2, 2)))\nmodel.add(Dropout(0.25))\n\nmodel.add(Flatten())\nmodel.add(Dense(64*5*5, 512, init='normal'))\nmodel.add(Activation('tanh'))\n\nmodel.add(Dense(512, 10, init='normal'))\nmodel.add(Activation('softmax'))\n\nsgd = SGD(l2=0.001,lr=0.0065, decay=1e-6, momentum=0.9, nesterov=True)\nmodel.compile(loss='categorical_crossentropy', optimizer=sgd,class_mode=\"categorical\")\n\n#checkpointer = ModelCheckpoint(filepath=\"weight.hdf5\",verbose=1,save_best_only=True)\n#model.fit(data, label, batch_size=100,nb_epoch=10,shuffle=True,verbose=1,show_accuracy=True,validation_split=0.2,callbacks=[checkpointer])\nresult = model.fit(data, label, batch_size=50,nb_epoch=35,shuffle=True,verbose=1,show_accuracy=True,validation_split=0.2)\n#model.save_weights(weights,accuracy=False)\n\n# plot the result\n\nplt.figure\nplt.plot(result.epoch,result.history['acc'],label=\"acc\")\nplt.plot(result.epoch,result.history['val_acc'],label=\"val_acc\")\nplt.scatter(result.epoch,result.history['acc'],marker='*')\nplt.scatter(result.epoch,result.history['val_acc'])\nplt.legend(loc='under right')\nplt.show()\n\nplt.figure\nplt.plot(result.epoch,result.history['loss'],label=\"loss\")\nplt.plot(result.epoch,result.history['val_loss'],label=\"val_loss\")\nplt.scatter(result.epoch,result.history['loss'],marker='*')\nplt.scatter(result.epoch,result.history['val_loss'],marker='*')\nplt.legend(loc='upper right')\nplt.show()\n" }, { "path": "DeepLearning/CNN_mnist/cnn.py", "content": "# -*- coding: utf-8 -*-\n\nfrom __future__ import absolute_import\nfrom __future__ import print_function\nfrom keras.preprocessing.image import ImageDataGenerator\nfrom keras.models import Sequential\nfrom keras.layers.core import Dense, Dropout, Activation, Flatten\nfrom keras.layers.advanced_activations import PReLU\nfrom keras.layers.convolutional import Convolution2D, MaxPooling2D\nfrom keras.optimizers import SGD, Adadelta, Adagrad\nfrom keras.utils import np_utils, generic_utils\nfrom data import load_data\n\nimport matplotlib.pyplot as plt\n\n'''\ntrain the dataset of face 42000 image\nlearning rate : 0.0015\npathc size:100\ndropout :0.25\nl2 = 0.01\n'''\n\ndef funcnn(LR,BS):\n data, label = load_data()\n label = np_utils.to_categorical(label, 10)\n \n model = Sequential()\n \n model.add(Convolution2D(4, 1, 5, 5, border_mode='valid')) \n model.add(Activation('relu'))\n model.add(Dropout(0.25))\n \n model.add(Convolution2D(8,4, 3, 3, border_mode='valid'))\n model.add(Activation('relu'))\n model.add(MaxPooling2D(poolsize=(2, 2)))\n model.add(Dropout(0.25))\n \n model.add(Convolution2D(16, 8, 3, 3, border_mode='valid')) \n model.add(Activation('relu'))\n model.add(MaxPooling2D(poolsize=(2, 2)))\n model.add(Dropout(0.25))\n \n model.add(Flatten())\n model.add(Dense(16*4*4, 256, init='normal'))\n model.add(Activation('tanh'))\n \n model.add(Dense(256, 10, init='normal'))\n model.add(Activation('softmax'))\n \n sgd = SGD(l2=0.001,lr=LR, decay=1e-6, momentum=0.9, nesterov=True)\n model.compile(loss='categorical_crossentropy', optimizer=sgd,class_mode=\"categorical\")\n \n #checkpointer = ModelCheckpoint(filepath=\"weight.hdf5\",verbose=1,save_best_only=True)\n #model.fit(data, label, batch_size=100,nb_epoch=10,shuffle=True,verbose=1,show_accuracy=True,validation_split=0.2,callbacks=[checkpointer])\n result = model.fit(data, label, batch_size=BS,nb_epoch=20,shuffle=True,verbose=1,show_accuracy=True,validation_split=0.2)\n #model.save_weights(weights,accuracy=False)\n \n # plot the result\n \n plt.figure\n plt.plot(result.epoch,result.history['acc'],label=\"acc\")\n plt.plot(result.epoch,result.history['val_acc'],label=\"val_acc\")\n plt.scatter(result.epoch,result.history['acc'],marker='*')\n plt.scatter(result.epoch,result.history['val_acc'])\n plt.legend(loc='under right')\n plt.show()\n \n plt.figure\n plt.plot(result.epoch,result.history['loss'],label=\"loss\")\n plt.plot(result.epoch,result.history['val_loss'],label=\"val_loss\")\n plt.scatter(result.epoch,result.history['loss'],marker='*')\n plt.scatter(result.epoch,result.history['val_loss'],marker='*')\n plt.legend(loc='upper right')\n plt.show()\n \n\n" }, { "path": "DeepLearning/CNN_mnist/data.py", "content": "# -*- coding: utf-8 -*-\nimport os\nfrom PIL import Image\nimport numpy as np\n\ndef load_data():\n print 'start loading data...'\n data = np.empty((42000,1,28,28),dtype=\"float32\")\n label = np.empty((42000,),dtype=\"uint8\")\n\n imgs = os.listdir(\"./mnist\")\n num = len(imgs)\n for i in range(num):\n img = Image.open(\"./mnist/\"+imgs[i])\n arr = np.asarray(img,dtype=\"float32\")\n data[i,:,:,:] = arr\n label[i] = int(imgs[i].split('.')[0])\n return data,label\n\n" }, { "path": "DeepLearning/CNN_mnist/trainCNN.py", "content": "# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Tue Aug 25 10:59:38 2015\n\n@author: lab-liu.longpo\n\"\"\"\n\nfrom cnn import funcnn\nimport numpy as np\n\n\ndef floatrange(start,stop,steps):\n return [start+float(i) * (stop-start)/(float(steps)-1) for i in range(steps)]\n\n\n#LR = [0.03]\n#LR = floatrange(0.033,0.036,10)\nLR = [0.0015]\nBS = [100]\n#result = np.empty((3,8,10),dtype=\"float32\")\nk = 0;\nfor i in range(1):\n for j in range(1):\n print 'test',k\n k = k+1\n tmp = funcnn(LR[i],BS[j])\n #result[i,j,:] = tmp.history['val_acc']\n print 'learning rate:',LR[i],'batch size:',BS[j]\n \n" }, { "path": "DeepLearning/UFLDL/Vectorization_sparseae_exercise/checkNumericalGradient.m", "content": "function [] = checkNumericalGradient()\n% This code can be used to check your numerical gradient implementation \n% in computeNumericalGradient.m\n% It analytically evaluates the gradient of a very simple function called\n% simpleQuadraticFunction (see below) and compares the result with your numerical\n% solution. Your numerical gradient implementation is incorrect if\n% your numerical solution deviates too much from the analytical solution.\n \n% Evaluate the function and gradient at x = [4; 10]; (Here, x is a 2d vector.)\nx = [4; 10];\n[value, grad] = simpleQuadraticFunction(x);\n\n% Use your code to numerically compute the gradient of simpleQuadraticFunction at x.\n% (The notation \"@simpleQuadraticFunction\" denotes a pointer to a function.)\nnumgrad = computeNumericalGradient(@simpleQuadraticFunction, x);\n\n% Visually examine the two gradient computations. The two columns\n% you get should be very similar. \ndisp([numgrad grad]);\nfprintf('The above two columns you get should be very similar.\\n(Left-Your Numerical Gradient, Right-Analytical Gradient)\\n\\n');\n\n% Evaluate the norm of the difference between two solutions. \n% If you have a correct implementation, and assuming you used EPSILON = 0.0001 \n% in computeNumericalGradient.m, then diff below should be 2.1452e-12 \ndiff = norm(numgrad-grad)/norm(numgrad+grad);\ndisp(diff); \nfprintf('Norm of the difference between numerical and analytical gradient (should be < 1e-9)\\n\\n');\nend\n\n\n \nfunction [value,grad] = simpleQuadraticFunction(x)\n% this function accepts a 2D vector as input. \n% Its outputs are:\n% value: h(x1, x2) = x1^2 + 3*x1*x2\n% grad: A 2x1 vector that gives the partial derivatives of h with respect to x1 and x2 \n% Note that when we pass @simpleQuadraticFunction(x) to computeNumericalGradients, we're assuming\n% that computeNumericalGradients will use only the first returned value of this function.\n\nvalue = x(1)^2 + 3*x(1)*x(2);\n\ngrad = zeros(2, 1);\n% 对X1 求偏导\ngrad(1) = 2*x(1) + 3*x(2);\n% 对X2 求偏导\ngrad(2) = 3*x(1);\n\nend\n" }, { "path": "DeepLearning/UFLDL/Vectorization_sparseae_exercise/computeNumericalGradient.m", "content": "function numgrad = computeNumericalGradient(J, theta)\n% numgrad = computeNumericalGradient(J, theta)\n% theta: a vector of parameters\n% J: a function that outputs a real-number. Calling y = J(theta) will return the\n% function value at theta. \n \n% Initialize numgrad with zeros 2x1 这是输入X的行数，其实也就是参数W的个数\nnumgrad = zeros(size(theta));\n\n%% ---------- YOUR CODE HERE --------------------------------------\n% Instructions: \n% Implement numerical gradient checking, and return the result in numgrad. \n% (See Section 2.3 of the lecture notes.)\n% You should write code so that numgrad(i) is (the numerical approximation to) the \n% partial derivative of J with respect to the i-th input argument, evaluated at theta. \n% I.e., numgrad(i) should be the (approximately) the partial derivative of J with \n% respect to theta(i).\n% \n% Hint: You will probably want to compute the elements of numgrad one at a time. \n\n\nepsilon = 0.000001;\n% 2 x2 得到参数的个数\nnumW = size(theta,1);\nI = eye(numW);\nI = I * epsilon;\nfor j = 1:numW\n numgrad(j) = (J(theta+I(:,j)) - J(theta-I(:,j))) / (2 * epsilon);\nend\n\n\n\n\n\n\n%% ---------------------------------------------------------------\nend\n" }, { "path": "DeepLearning/UFLDL/Vectorization_sparseae_exercise/display_network.m", "content": "function [h, array] = display_network(A, opt_normalize, opt_graycolor, cols, opt_colmajor)\n% This function visualizes filters in matrix A. Each column of A is a\n% filter. We will reshape each column into a square image and visualizes\n% on each cell of the visualization panel. \n% All other parameters are optional, usually you do not need to worry\n% about it.\n% opt_normalize: whether we need to normalize the filter so that all of\n% them can have similar contrast. Default value is true.\n% opt_graycolor: whether we use gray as the heat map. Default is true.\n% cols: how many columns are there in the display. Default value is the\n% squareroot of the number of columns in A.\n% opt_colmajor: you can switch convention to row major for A. In that\n% case, each row of A is a filter. Default value is false.\nwarning off all\n\nif ~exist('opt_normalize', 'var') || isempty(opt_normalize)\n opt_normalize= true;\nend\n\nif ~exist('opt_graycolor', 'var') || isempty(opt_graycolor)\n opt_graycolor= true;\nend\n\nif ~exist('opt_colmajor', 'var') || isempty(opt_colmajor)\n opt_colmajor = false;\nend\n\n% rescale\nA = A - mean(A(:));\n\nif opt_graycolor, colormap(gray); end\n\n% compute rows, cols\n[L M]=size(A);\nsz=sqrt(L);\nbuf=1;\nif ~exist('cols', 'var')\n if floor(sqrt(M))^2 ~= M\n n=ceil(sqrt(M));\n while mod(M, n)~=0 && n<1.2*sqrt(M), n=n+1; end\n m=ceil(M/n);\n else\n n=sqrt(M);\n m=n;\n end\nelse\n n = cols;\n m = ceil(M/n);\nend\n\narray=-ones(buf+m*(sz+buf),buf+n*(sz+buf));\n\nif ~opt_graycolor\n array = 0.1.* array;\nend\n\n\nif ~opt_colmajor\n k=1;\n for i=1:m\n for j=1:n\n if k>M, \n continue; \n end\n clim=max(abs(A(:,k)));\n if opt_normalize\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz)/clim;\n else\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz)/max(abs(A(:)));\n end\n k=k+1;\n end\n end\nelse\n k=1;\n for j=1:n\n for i=1:m\n if k>M, \n continue; \n end\n clim=max(abs(A(:,k)));\n if opt_normalize\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz)/clim;\n else\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz);\n end\n k=k+1;\n end\n end\nend\n\nif opt_graycolor\n h=imagesc(array,'EraseMode','none',[-1 1]);\nelse\n h=imagesc(array,'EraseMode','none',[-1 1]);\nend\naxis image off\n\ndrawnow;\n\nwarning on all\n" }, { "path": "DeepLearning/UFLDL/Vectorization_sparseae_exercise/initializeParameters.m", "content": "function theta = initializeParameters(hiddenSize, visibleSize)\n\n%% Initialize parameters randomly based on layer sizes.\nr = sqrt(6) / sqrt(hiddenSize+visibleSize+1); % we'll choose weights uniformly from the interval [-r, r]\nW1 = rand(hiddenSize, visibleSize) * 2 * r - r;\nW2 = rand(visibleSize, hiddenSize) * 2 * r - r;\n\nb1 = zeros(hiddenSize, 1);\nb2 = zeros(visibleSize, 1);\n\n% Convert weights and bias gradients to the vector form.\n% This step will \"unroll\" (flatten and concatenate together) all \n% your parameters into a vector, which can then be used with minFunc. \ntheta = [W1(:) ; W2(:) ; b1(:) ; b2(:)];\n\nend\n\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/display_network.m", "content": "function [h, array] = display_network(A, opt_normalize, opt_graycolor, cols, opt_colmajor)\n% This function visualizes filters in matrix A. Each column of A is a\n% filter. We will reshape each column into a square image and visualizes\n% on each cell of the visualization panel. \n% All other parameters are optional, usually you do not need to worry\n% about it.\n% opt_normalize: whether we need to normalize the filter so that all of\n% them can have similar contrast. Default value is true.\n% opt_graycolor: whether we use gray as the heat map. Default is true.\n% cols: how many columns are there in the display. Default value is the\n% squareroot of the number of columns in A.\n% opt_colmajor: you can switch convention to row major for A. In that\n% case, each row of A is a filter. Default value is false.\nwarning off all\n\nif ~exist('opt_normalize', 'var') || isempty(opt_normalize)\n opt_normalize= true;\nend\n\nif ~exist('opt_graycolor', 'var') || isempty(opt_graycolor)\n opt_graycolor= true;\nend\n\nif ~exist('opt_colmajor', 'var') || isempty(opt_colmajor)\n opt_colmajor = false;\nend\n\n% rescale\nA = A - mean(A(:));\n\nif opt_graycolor, colormap(gray); end\n\n% compute rows, cols\n[L M]=size(A);\nsz=sqrt(L);\nbuf=1;\nif ~exist('cols', 'var')\n if floor(sqrt(M))^2 ~= M\n n=ceil(sqrt(M));\n while mod(M, n)~=0 && n<1.2*sqrt(M), n=n+1; end\n m=ceil(M/n);\n else\n n=sqrt(M);\n m=n;\n end\nelse\n n = cols;\n m = ceil(M/n);\nend\n\narray=-ones(buf+m*(sz+buf),buf+n*(sz+buf));\n\nif ~opt_graycolor\n array = 0.1.* array;\nend\n\n\nif ~opt_colmajor\n k=1;\n for i=1:m\n for j=1:n\n if k>M, \n continue; \n end\n clim=max(abs(A(:,k)));\n if opt_normalize\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz)/clim;\n else\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz)/max(abs(A(:)));\n end\n k=k+1;\n end\n end\nelse\n k=1;\n for j=1:n\n for i=1:m\n if k>M, \n continue; \n end\n clim=max(abs(A(:,k)));\n if opt_normalize\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz)/clim;\n else\n array(buf+(i-1)*(sz+buf)+(1:sz),buf+(j-1)*(sz+buf)+(1:sz))=reshape(A(:,k),sz,sz);\n end\n k=k+1;\n end\n end\nend\n\nif opt_graycolor\n h=imagesc(array,'EraseMode','none',[-1 1]);\nelse\n h=imagesc(array,'EraseMode','none',[-1 1]);\nend\naxis image off\n\ndrawnow;\n\nwarning on all\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/feedForwardAutoencoder.m", "content": "function [activation] = feedForwardAutoencoder(theta, hiddenSize, visibleSize, data)\n\n% theta: trained weights from the autoencoder\n% visibleSize: the number of input units (probably 64) \n% hiddenSize: the number of hidden units (probably 25) \n% data: Our matrix containing the training data as columns. So, data(:,i) is the i-th training example. \n \n% We first convert theta to the (W1, W2, b1, b2) matrix/vector format, so that this \n% follows the notation convention of the lecture notes. \n%200 x 784\nW1 = reshape(theta(1:hiddenSize*visibleSize), hiddenSize, visibleSize);\n% 200 x 1\nb1 = theta(2*hiddenSize*visibleSize+1:2*hiddenSize*visibleSize+hiddenSize);\n%% ---------- YOUR CODE HERE --------------------------------------\n% Instructions: Compute the activation of the hidden layer for the Sparse Autoencoder.\nactivation = sigmoid(bsxfun(@plus,W1 * data,b1)) ;\n%-------------------------------------------------------------------\n\nend\n\n%-------------------------------------------------------------------\n% Here's an implementation of the sigmoid function, which you may find useful\n% in your computation of the costs and the gradients. This inputs a (row or\n% column) vector (say (z1, z2, z3)) and returns (f(z1), f(z2), f(z3)). \n\nfunction sigm = sigmoid(x)\n sigm = 1 ./ (1 + exp(-x));\nend\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/initializeParameters.m", "content": "function theta = initializeParameters(hiddenSize, visibleSize)\n\n%% Initialize parameters randomly based on layer sizes.\nr = sqrt(6) / sqrt(hiddenSize+visibleSize+1); % we'll choose weights uniformly from the interval [-r, r]\nW1 = rand(hiddenSize, visibleSize) * 2 * r - r;\nW2 = rand(visibleSize, hiddenSize) * 2 * r - r;\n\nb1 = zeros(hiddenSize, 1);\nb2 = zeros(visibleSize, 1);\n\n% Convert weights and bias gradients to the vector form.\n% This step will \"unroll\" (flatten and concatenate together) all \n% your parameters into a vector, which can then be used with minFunc. \ntheta = [W1(:) ; W2(:) ; b1(:) ; b2(:)];\n\nend\n\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/loadMNISTImages.m", "content": "function images = loadMNISTImages(filename)\n%loadMNISTImages returns a 28x28x[number of MNIST images] matrix containing\n%the raw MNIST images\n\nfp = fopen(filename, 'rb');\nassert(fp ~= -1, ['Could not open ', filename, '']);\n\nmagic = fread(fp, 1, 'int32', 0, 'ieee-be');\nassert(magic == 2051, ['Bad magic number in ', filename, '']);\n\nnumImages = fread(fp, 1, 'int32', 0, 'ieee-be');\nnumRows = fread(fp, 1, 'int32', 0, 'ieee-be');\nnumCols = fread(fp, 1, 'int32', 0, 'ieee-be');\n\nimages = fread(fp, inf, 'unsigned char');\nimages = reshape(images, numCols, numRows, numImages);\nimages = permute(images,[2 1 3]);\n\nfclose(fp);\n\n% Reshape to #pixels x #examples\nimages = reshape(images, size(images, 1) * size(images, 2), size(images, 3));\n% Convert to double and rescale to [0,1]\nimages = double(images) / 255;\n\nend\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/loadMNISTLabels.m", "content": "function labels = loadMNISTLabels(filename)\n%loadMNISTLabels returns a [number of MNIST images]x1 matrix containing\n%the labels for the MNIST images\n\nfp = fopen(filename, 'rb');\nassert(fp ~= -1, ['Could not open ', filename, '']);\n\nmagic = fread(fp, 1, 'int32', 0, 'ieee-be');\nassert(magic == 2049, ['Bad magic number in ', filename, '']);\n\nnumLabels = fread(fp, 1, 'int32', 0, 'ieee-be');\n\nlabels = fread(fp, inf, 'unsigned char');\n\nassert(size(labels,1) == numLabels, 'Mismatch in label count');\n\nfclose(fp);\n\nend\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/softmaxCost.m", "content": "function [cost, grad] = softmaxCost(theta, numClasses, inputSize, lambda, data, labels)\n\n% numClasses - the number of classes \n% inputSize - the size N of the input vector\n% lambda - weight decay parameter\n% data - the N x M input matrix, where each column data(:, i) corresponds to\n% a single test set\n% labels - an M x 1 matrix containing the labels corresponding for the input data\n%\n\n% Unroll the parameters from theta\n% 10x8\ntheta = reshape(theta, numClasses, inputSize);\n%100 data 8x100\nnumCases = size(data, 2);\n%sparse(r,c,v) = [r(i),c(i)] = v(i)\n% 每一列的第 label 行 设置为i，也就是，每个样本属于哪个label，该位置就是1，其他的均为0\n% 10 x 100\ngroundTruth = full(sparse(labels, 1:numCases, 1));\ncost = 0;\n%10 x 8\nthetagrad = zeros(numClasses, inputSize);\n\n%% ---------- YOUR CODE HERE --------------------------------------\n% Instructions: Compute the cost and gradient for softmax regression.\n% You need to compute thetagrad and cost.\n% The groundTruth matrix might come in handy.\n\n% 10x100 10x8 8x100 100个样本，每个样本属于每个label的概率\nM = theta * data;\nM = exp(bsxfun(@minus,M,max(M,[],1)));\nH = bsxfun(@rdivide,M,sum(M));\ncost = - sum(sum((groundTruth .* log(H)))) / size(data,2) + lambda * sum(sum(theta.^2)) / 2 ;\n% 10x8 10x100 100x8\nthetagrad = - ((groundTruth - H)) * data' /size(data,2) +lambda * theta;\n\n\n\n\n\n\n\n\n\n% ------------------------------------------------------------------\n% Unroll the gradient matrices into a vector for minFunc\ngrad = [thetagrad(:)];\nend\n\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/softmaxPredict.m", "content": "function [pred] = softmaxPredict(softmaxModel, data)\n\n% softmaxModel - model trained using softmaxTrain\n% data - the N x M input matrix, where each column data(:, i) corresponds to\n% a single test set\n%\n% Your code should produce the prediction matrix \n% pred, where pred(i) is argmax_c P(y(c) | x(i)).\n \n% Unroll the parameters from theta\ntheta = softmaxModel.optTheta; % this provides a numClasses x inputSize matrix\npred = zeros(1, size(data, 2));\n\n%% ---------- YOUR CODE HERE --------------------------------------\n% Instructions: Compute pred using theta assuming that the labels start \n% from 1.\n[Val , pred] = max(theta*data);\n\n\n\n\n\n\n% ---------------------------------------------------------------------\n\nend\n\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/softmaxTrain.m", "content": "function [softmaxModel] = softmaxTrain(inputSize, numClasses, lambda, inputData, labels, options)\n%softmaxTrain Train a softmax model with the given parameters on the given\n% data. Returns softmaxOptTheta, a vector containing the trained parameters\n% for the model.\n%\n% inputSize: the size of an input vector x^(i)\n% numClasses: the number of classes \n% lambda: weight decay parameter\n% inputData: an N by M matrix containing the input data, such that\n% inputData(:, c) is the cth input\n% labels: M by 1 matrix containing the class labels for the\n% corresponding inputs. labels(c) is the class label for\n% the cth input\n% options (optional): options\n% options.maxIter: number of iterations to train for\n\nif ~exist('options', 'var')\n options = struct;\nend\n\nif ~isfield(options, 'maxIter')\n options.maxIter = 400;\nend\n\n% initialize parameters\ntheta = 0.005 * randn(numClasses * inputSize, 1);\n\n% Use minFunc to minimize the function\naddpath minFunc/\noptions.Method = 'lbfgs'; % Here, we use L-BFGS to optimize our cost\n % function. Generally, for minFunc to work, you\n % need a function pointer with two outputs: the\n % function value and the gradient. In our problem,\n % softmaxCost.m satisfies this.\nminFuncOptions.display = 'on';\n\n[softmaxOptTheta, cost] = minFunc( @(p) softmaxCost(p, ...\n numClasses, inputSize, lambda, ...\n inputData, labels), ... \n theta, options);\n\n% Fold softmaxOptTheta into a nicer format\nsoftmaxModel.optTheta = reshape(softmaxOptTheta, numClasses, inputSize);\nsoftmaxModel.inputSize = inputSize;\nsoftmaxModel.numClasses = numClasses;\n \nend \n" }, { "path": "DeepLearning/UFLDL/stl_exercise/sparseAutoencoderCost.m", "content": "function [cost,grad] = sparseAutoencoderCost(theta, visibleSize, hiddenSize, ...\n lambda, sparsityParam, beta, data)\n%lambda = 0;\n%beta = 0;\n% visibleSize: the number of input units (probably 64) \n% hiddenSize: the number of hidden units (probably 25) \n% lambda: weight decay parameter\n% sparsityParam: The desired average activation for the hidden units (denoted in the lecture\n% notes by the greek alphabet rho, which looks like a lower-case \"p\").\n% beta: weight of sparsity penalty term\n% data: Our 64x10000 matrix containing the training data. So, data(:,i) is the i-th training example. \n \n% The input theta is a vector (because minFunc expects the parameters to be a vector). \n\n% We first convert theta to the (W1, W2, b1, b2) matrix/vector format, so that this \n% follows the notation convention of the lecture notes. \n\n% 学习率 自己定义的\nalpha = 0.03;\n\n% 计算隐藏层神经元的激活度\np = zeros(hiddenSize,1);\n\nW1 = reshape(theta(1:hiddenSize*visibleSize), hiddenSize, visibleSize);\nW2 = reshape(theta(hiddenSize*visibleSize+1:2*hiddenSize*visibleSize), visibleSize, hiddenSize);\nb1 = theta(2*hiddenSize*visibleSize+1:2*hiddenSize*visibleSize+hiddenSize);\nb2 = theta(2*hiddenSize*visibleSize+hiddenSize+1:end);\n\n% Cost and gradient variables (your code needs to compute these values). \n% Here, we initialize them to zeros. \n\n%% ---------- YOUR CODE HERE --------------------------------------\n% Instructions: Compute the cost/optimization objective J_sparse(W,b) for the Sparse Autoencoder,\n% and the corresponding gradients W1grad, W2grad, b1grad, b2grad.\n%\n% W1grad, W2grad, b1grad and b2grad should be computed using backpropagation.\n% Note that W1grad has the same dimensions as W1, b1grad has the same dimensions\n% as b1, etc. Your code should set W1grad to be the partial derivative of J_sparse(W,b) with\n% respect to W1. I.e., W1grad(i,j) should be the partial derivative of J_sparse(W,b) \n% with respect to the input parameter W1(i,j). Thus, W1grad should be equal to the term \n% [(1/m) \\Delta W^{(1)} + \\lambda W^{(1)}] in the last block of pseudo-code in Section 2.2 \n% of the lecture notes (and similarly for W2grad, b1grad, b2grad).\n% \n% Stated differently, if we were using batch gradient descent to optimize the parameters,\n% the gradient descent update to W1 would be W1 := W1 - alpha * W1grad, and similarly for W2, b1, b2. \n% \nnumPatches = size(data,2);\nKLdist = 0;\n\n%% 向前传输\n\na2 = sigmoid(W1*data+repmat(b1,[1,numPatches]));\np = sum(a2,2);\na3 = sigmoid(W2 * a2 + repmat(b2,[1,numPatches]));\nJ_sparse = 0.5 * sum(sum((a3-data).^2));\n\n%% 计算 隐藏层的平均激活度\np = p / numPatches ;\n\n%% 向后传输 \n\n residual3 = -(data-a3).*a3.*(1-a3);\n tmp = beta * ( - sparsityParam ./ p + (1-sparsityParam) ./ (1-p));\n residual2 = (W2' * residual3 + repmat(tmp,[1,numPatches])) .* a2.*(1-a2);\n \n W2grad = residual3 * a2' / numPatches + lambda * W2 ;\n W1grad = residual2 * data' / numPatches + lambda * W1 ;\n b2grad = sum(residual3,2) / numPatches; \n b1grad = sum(residual2,2) / numPatches; \n\n%% 更新权重参数 加上 lambda 权重衰减\nW2 = W2 - alpha * ( W2grad );\nW1 = W1 - alpha * ( W1grad );\n\nb2 = b2 - alpha * (b2grad );\nb1 = b1 - alpha * (b1grad );\n\n%% 计算KL相对熵\nfor j = 1:hiddenSize\n KLdist = KLdist + sparsityParam *log( sparsityParam / p(j) ) + (1 - sparsityParam) * log((1-sparsityParam) / (1 - p(j)));\nend\n\n%% costFunction 加上 lambda 权重衰减\ncost = J_sparse / numPatches + (sum(sum(W1.^2)) + sum(sum(W2.^2))) * lambda / 2 + beta * KLdist;\n\n%-------------------------------------------------------------------\n% After computing the cost and gradient, we will convert the gradients back\n% to a vector format (suitable for minFunc). Specifically, we will unroll\n% your gradient matrices into a vector.\n\ngrad = [W1grad(:) ; W2grad(:) ; b1grad(:) ; b2grad(:)];\n\nend\n\n%-------------------------------------------------------------------\n% Here's an implementation of the sigmoid function, which you may find useful\n% in your computation of the costs and the gradients. This inputs a (row or\n% column) vector (say (z1, z2, z3)) and returns (f(z1), f(z2), f(z3)). \n\nfunction sigm = sigmoid(x)\n \n sigm = 1 ./ (1 + exp(-x));\nend\n\n\n" }, { "path": "DeepLearning/UFLDL/stl_exercise/stlExercise.m", "content": "%% CS294A/CS294W Self-taught Learning Exercise\n\n% Instructions\n% ------------\n% \n% This file contains code that helps you get started on the\n% self-taught learning. You will need to complete code in feedForwardAutoencoder.m\n% You will also need to have implemented sparseAutoencoderCost.m and \n% softmaxCost.m from previous exercises.\n%\n%% ======================================================================\n% STEP 0: Here we provide the relevant parameters values that will\n% allow your sparse autoencoder to get good filters; you do not need to \n% change the parameters below.\nclc\nclear \n\ninputSize = 28 * 28;\nnumLabels = 5;\nhiddenSize = 200;\nsparsityParam = 0.1; % desired average activation of the hidden units.\n % (This was denoted by the Greek alphabet rho, which looks like a lower-case \"p\",\n\t\t % in the lecture notes). \nlambda = 3e-3; % weight decay parameter \nbeta = 3; % weight of sparsity penalty term \nmaxIter = 400;\n\n%% ======================================================================\n% STEP 1: Load data from the MNIST database\n%\n% This loads our training and test data from the MNIST database files.\n% We have sorted the data for you in this so that you will not have to\n% change it.\n\n% Load MNIST database files\nmnistData = loadMNISTImages('train-images.idx3-ubyte');\nmnistLabels = loadMNISTLabels('train-labels.idx1-ubyte');\n\n% Set Unlabeled Set (All Images)\n\n% Simulate a Labeled and Unlabeled set\nlabeledSet = find(mnistLabels >= 0 & mnistLabels <= 4);\nunlabeledSet = find(mnistLabels >= 5);\n\nnumTrain = round(numel(labeledSet)/2);\ntrainSet = labeledSet(1:numTrain);\ntestSet = labeledSet(numTrain+1:end);\n\nunlabeledData = mnistData(:, unlabeledSet);\n\ntrainData = mnistData(:, trainSet);\ntrainLabels = mnistLabels(trainSet)' + 1; % Shift Labels to the Range 1-5\n\ntestData = mnistData(:, testSet);\ntestLabels = mnistLabels(testSet)' + 1; % Shift Labels to the Range 1-5\n\n% Output Some Statistics\nfprintf('# examples in unlabeled set: %d\\n', size(unlabeledData, 2));\nfprintf('# examples in supervised training set: %d\\n\\n', size(trainData, 2));\nfprintf('# examples in supervised testing set: %d\\n\\n', size(testData, 2));\n\n%% ======================================================================\n% STEP 2: Train the sparse autoencoder\n% This trains the sparse autoencoder on the unlabeled training\n% images. \n\n% Randomly initialize the parameters\ntheta = initializeParameters(hiddenSize, inputSize);\n\n%% ----------------- YOUR CODE HERE ----------------------\n% Find opttheta by running the sparse autoencoder on\n% unlabeledTrainingImages\n\naddpath minFunc/\noptions.Method = 'lbfgs'; \noptions.maxIter = 400; \noptions.display = 'on'; \nvisibleSize = inputSize;\n[opttheta, cost] = minFunc( @(p) sparseAutoencoderCost(p, ...\n visibleSize, hiddenSize, ...\n lambda, sparsityParam, ...\n beta, unlabeledData), ...\n theta, options);\n\n%% -----------------------------------------------------\n \n% Visualize weights\nW1 = reshape(opttheta(1:hiddenSize * inputSize), hiddenSize, inputSize);\ndisplay_network(W1');\n\n%%======================================================================\n%% STEP 3: Extract Features from the Supervised Dataset\n% \n% You need to complete the code in feedForwardAutoencoder.m so that the \n% following command will extract features from the data.\n\ntrainFeatures = feedForwardAutoencoder(opttheta, hiddenSize, inputSize, ...\n trainData);\n\ntestFeatures = feedForwardAutoencoder(opttheta, hiddenSize, inputSize, ...\n testData);\n\n%%======================================================================\n%% STEP 4: Train the softmax classifier\n\nsoftmaxModel = struct; \n%% ----------------- YOUR CODE HERE ----------------------\n% Use softmaxTrain.m from the previous exercise to train a multi-class\n% classifier. \noptions.maxIter = 100;\nlambda = 1e-4;\nnumClasses = 5;\ninputData = trainFeatures;\nlabels = trainLabels ;\ninputSize = hiddenSize; \nsoftmaxModel = softmaxTrain(inputSize, numClasses, lambda, inputData, labels, options);\n% Use lambda = 1e-4 for the weight regularization for softmax\n\n% You need to compute softmaxModel using softmaxTrain on trainFeatures and\n% trainLabels\n\n%% -----------------------------------------------------\n\n\n%%======================================================================\n%% STEP 5: Testing \n\n%% ----------------- YOUR CODE HERE ----------------------\n% Compute Predictions on the test set (testFeatures) using softmaxPredict\n% and softmaxModel\n\n[pred] = softmaxPredict(softmaxModel, testFeatures);\n\n%% -----------------------------------------------------\n\n% Classification Score\nfprintf('Test Accuracy: %f%%\\n', 100*mean(pred(:) == testLabels(:)));\n\n% (note that we shift the labels by 1, so that digit 0 now corresponds to\n% label 1)\n%\n% Accuracy is the proportion of correctly classified images\n% The results for our implementation was:\n%\n% Accuracy: 98.3%\n%\n% \n" }, { "path": "GMM/README.md", "content": "博客链接：http://blog.csdn.net/llp1992/article/details/47058109\n\n更多机器学习深度学习博客请关注CSDN博客：[LiuLongpo](http://blog.csdn.net/llp1992)\n" }, { "path": "GMM/gmm.m", "content": "function varargout = gmm(X, K_or_centroids)\n% ============================================================\n% Expectation-Maximization iteration implementation of\n% Gaussian Mixture Model.\n%\n% PX = GMM(X, K_OR_CENTROIDS)\n% [PX MODEL] = GMM(X, K_OR_CENTROIDS)\n%\n% - X: N-by-D data matrix.\n% - K_OR_CENTROIDS: either K indicating the number of\n% components or a K-by-D matrix indicating the\n% choosing of the initial K centroids.\n%\n% - PX: N-by-K matrix indicating the probability of each\n% component generating each point.\n% - MODEL: a structure containing the parameters for a GMM:\n% MODEL.Miu: a K-by-D matrix.\n% MODEL.Sigma: a D-by-D-by-K matrix.\n% MODEL.Pi: a 1-by-K vector.\n% ============================================================\n\nthreshold = 1e-15;\n[N, D] = size(X);\n% isscalar 判断是否为标量\nif isscalar(K_or_centroids)\n K = K_or_centroids;\n % randomly pick centroids\n rndp = randperm(N);\n centroids = X(rndp(1:K), :);\nelse % 矩阵，给出每一类的初始化\n K = size(K_or_centroids, 1);\n centroids = K_or_centroids;\nend\n\n% initial values\n[pMiu pPi pSigma] = init_params();\n\nLprev = -inf;\nwhile true\n %% Estiamtion Step\n Px = calc_prob();\n\n % new value for pGamma\n pGamma = Px .* repmat(pPi, N, 1);\n pGamma = pGamma ./ repmat(sum(pGamma, 2), 1, K);\n\n %% Maximization Step\n % new value for parameters of each Component\n Nk = sum(pGamma, 1);\n pMiu = diag(1./Nk) * pGamma' * X;\n pPi = Nk/N;\n for kk = 1:K\n Xshift = X-repmat(pMiu(kk, :), N, 1);\n pSigma(:, :, kk) = (Xshift' * (diag(pGamma(:, kk)) * Xshift)) / Nk(kk);\n end\n\n %% check for convergence\n L = sum(log(Px*pPi'));\n if L-Lprev < threshold\n break;\n end\n Lprev = L;\nend\n\n% 输出参数判定\nif nargout == 1\n varargout = {Px};\nelse\n model = [];\n model.Miu = pMiu;\n model.Sigma = pSigma;\n model.Pi = pPi;\n varargout = {Px, model};\nend\n\nfunction [pMiu pPi pSigma] = init_params()\n pMiu = centroids; % 均值，也就是K类的中心\n pPi = zeros(1, K); % 概率\n pSigma = zeros(D, D, K); %协方差矩阵，每个都是 D*D\n\n % hard assign x to each centroids \n % (X - pMiu)^2 = X^2 + pMiu^2 - 2*X*pMiu\n distmat = repmat(sum(X.*X, 2), 1, K) + repmat(sum(pMiu.*pMiu, 2)', N, 1) - 2*X*pMiu';\n [dummy labels] = min(distmat, [], 2);\n\n for k=1:K %初始化参数\n Xk = X(labels == k, :);\n pPi(k) = size(Xk, 1)/N;\n pSigma(:, :, k) = cov(Xk);\n end\nend\n\n% 计算概率\nfunction Px = calc_prob()\n Px = zeros(N, K);\n for k = 1:K\n Xshift = X-repmat(pMiu(k, :), N, 1);\n inv_pSigma = inv(pSigma(:, :, k)+diag(repmat(threshold,1,size(pSigma(:, :, k),1)))); % 方差矩阵求逆\n tmp = sum((Xshift*inv_pSigma) .* Xshift, 2);\n coef = (2*pi)^(-D/2) * sqrt(det(inv_pSigma)); % det 求方差矩阵的行列式 \n Px(:, k) = coef * exp(-0.5*tmp);\n end\nend\nend\n" }, { "path": "GMM/gmm.py", "content": "# -*- coding: utf-8 -*-\n\n'''\n Description ：GMM in Python\n Author ： LiuLongpo\n Time : 2015年7月26日16:54:48\n Source ：From pluskid\n'''\n\nimport sys;\nsys.path.append(\"E:\\Python\\MachineLearning\\PythonTools\")\nimport PythonUtils as pu\nimport matplotlib.pyplot as plt\nimport numpy as np\n'矩阵的逆矩阵需要的库'\nfrom numpy.linalg import *\n\ndef gmm(X,K):\n threshold = 1e-15\n N,D = np.shape(X)\n randV = pu.randIntList(1,N,K)\n centroids = X[randV]\n pMiu,pPi,pSigma = inti_params(centroids,K,X,N,D);\n Lprev = -np.inf\n while True:\n 'Estiamtion Step'\n Px = calc_prop(X,N,K,pMiu,pSigma,threshold,D)\n pGamma = Px * np.tile(pPi,(N,1))\n pGamma = pGamma / np.tile((np.sum(pGamma,axis=1)),(K,1)).T\n 'Maximization Step'\n Nk = np.sum(pGamma,axis=0)\n pMiu = np.dot(np.dot(np.diag(1 / Nk),pGamma.T),X)\n pPi = Nk / N\n for kk in range(K):\n Xshift = X - np.tile(pMiu[kk],(N,1))\n pSigma[:,:,kk] = (np.dot(np.dot(Xshift.T,np.diag(pGamma[:,kk])),Xshift)) / Nk[kk]\n\n 'check for convergence' \n L = np.sum(np.log(np.dot(Px,pPi.T)))\n if L-Lprev