[关闭]
@cleardusk 2015-11-27T09:18:54.000000Z 字数 10020 阅读 1178

单隐藏层网络实现代码(Python 版)

GjzCVCode


来源:https://github.com/mnielsen/neural-networks-and-deep-learning

  1. """network2.py
  2. ~~~~~~~~~~~~~~
  3. An improved version of network.py, implementing the stochastic
  4. gradient descent learning algorithm for a feedforward neural network.
  5. Improvements include the addition of the cross-entropy cost function,
  6. regularization, and better initialization of network weights. Note
  7. that I have focused on making the code simple, easily readable, and
  8. easily modifiable. It is not optimized, and omits many desirable
  9. features.
  10. """
  11. #### Libraries
  12. # Standard library
  13. import json
  14. import random
  15. import sys
  16. # Third-party libraries
  17. import numpy as np
  18. #### Define the quadratic and cross-entropy cost functions
  19. class QuadraticCost(object):
  20. @staticmethod
  21. def fn(a, y):
  22. """Return the cost associated with an output ``a`` and desired output
  23. ``y``.
  24. """
  25. return 0.5*np.linalg.norm(a-y)**2
  26. @staticmethod
  27. def delta(z, a, y):
  28. """Return the error delta from the output layer."""
  29. return (a-y) * sigmoid_prime(z)
  30. class CrossEntropyCost(object):
  31. @staticmethod
  32. def fn(a, y):
  33. """Return the cost associated with an output ``a`` and desired output
  34. ``y``. Note that np.nan_to_num is used to ensure numerical
  35. stability. In particular, if both ``a`` and ``y`` have a 1.0
  36. in the same slot, then the expression (1-y)*np.log(1-a)
  37. returns nan. The np.nan_to_num ensures that that is converted
  38. to the correct value (0.0).
  39. """
  40. return np.sum(np.nan_to_num(-y*np.log(a)-(1-y)*np.log(1-a)))
  41. @staticmethod
  42. def delta(z, a, y):
  43. """Return the error delta from the output layer. Note that the
  44. parameter ``z`` is not used by the method. It is included in
  45. the method's parameters in order to make the interface
  46. consistent with the delta method for other cost classes.
  47. """
  48. return (a-y)
  49. #### Main Network class
  50. class Network(object):
  51. def __init__(self, sizes, cost=CrossEntropyCost):
  52. """The list ``sizes`` contains the number of neurons in the respective
  53. layers of the network. For example, if the list was [2, 3, 1]
  54. then it would be a three-layer network, with the first layer
  55. containing 2 neurons, the second layer 3 neurons, and the
  56. third layer 1 neuron. The biases and weights for the network
  57. are initialized randomly, using
  58. ``self.default_weight_initializer`` (see docstring for that
  59. method).
  60. """
  61. self.num_layers = len(sizes)
  62. self.sizes = sizes
  63. self.default_weight_initializer()
  64. self.cost=cost
  65. def default_weight_initializer(self):
  66. """Initialize each weight using a Gaussian distribution with mean 0
  67. and standard deviation 1 over the square root of the number of
  68. weights connecting to the same neuron. Initialize the biases
  69. using a Gaussian distribution with mean 0 and standard
  70. deviation 1.
  71. Note that the first layer is assumed to be an input layer, and
  72. by convention we won't set any biases for those neurons, since
  73. biases are only ever used in computing the outputs from later
  74. layers.
  75. """
  76. self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]
  77. self.weights = [np.random.randn(y, x)/np.sqrt(x)
  78. for x, y in zip(self.sizes[:-1], self.sizes[1:])]
  79. def large_weight_initializer(self):
  80. """Initialize the weights using a Gaussian distribution with mean 0
  81. and standard deviation 1. Initialize the biases using a
  82. Gaussian distribution with mean 0 and standard deviation 1.
  83. Note that the first layer is assumed to be an input layer, and
  84. by convention we won't set any biases for those neurons, since
  85. biases are only ever used in computing the outputs from later
  86. layers.
  87. This weight and bias initializer uses the same approach as in
  88. Chapter 1, and is included for purposes of comparison. It
  89. will usually be better to use the default weight initializer
  90. instead.
  91. """
  92. self.biases = [np.random.randn(y, 1) for y in self.sizes[1:]]
  93. self.weights = [np.random.randn(y, x)
  94. for x, y in zip(self.sizes[:-1], self.sizes[1:])]
  95. def feedforward(self, a):
  96. """Return the output of the network if ``a`` is input."""
  97. for b, w in zip(self.biases, self.weights):
  98. a = sigmoid(np.dot(w, a)+b)
  99. return a
  100. def SGD(self, training_data, epochs, mini_batch_size, eta,
  101. lmbda = 0.0,
  102. evaluation_data=None,
  103. monitor_evaluation_cost=False,
  104. monitor_evaluation_accuracy=False,
  105. monitor_training_cost=False,
  106. monitor_training_accuracy=False):
  107. """Train the neural network using mini-batch stochastic gradient
  108. descent. The ``training_data`` is a list of tuples ``(x, y)``
  109. representing the training inputs and the desired outputs. The
  110. other non-optional parameters are self-explanatory, as is the
  111. regularization parameter ``lmbda``. The method also accepts
  112. ``evaluation_data``, usually either the validation or test
  113. data. We can monitor the cost and accuracy on either the
  114. evaluation data or the training data, by setting the
  115. appropriate flags. The method returns a tuple containing four
  116. lists: the (per-epoch) costs on the evaluation data, the
  117. accuracies on the evaluation data, the costs on the training
  118. data, and the accuracies on the training data. All values are
  119. evaluated at the end of each training epoch. So, for example,
  120. if we train for 30 epochs, then the first element of the tuple
  121. will be a 30-element list containing the cost on the
  122. evaluation data at the end of each epoch. Note that the lists
  123. are empty if the corresponding flag is not set.
  124. """
  125. if evaluation_data: n_data = len(evaluation_data)
  126. n = len(training_data)
  127. evaluation_cost, evaluation_accuracy = [], []
  128. training_cost, training_accuracy = [], []
  129. for j in xrange(epochs):
  130. random.shuffle(training_data)
  131. mini_batches = [
  132. training_data[k:k+mini_batch_size]
  133. for k in xrange(0, n, mini_batch_size)]
  134. for mini_batch in mini_batches:
  135. self.update_mini_batch(
  136. mini_batch, eta, lmbda, len(training_data))
  137. print "Epoch %s training complete" % j
  138. if monitor_training_cost:
  139. cost = self.total_cost(training_data, lmbda)
  140. training_cost.append(cost)
  141. print "Cost on training data: {}".format(cost)
  142. if monitor_training_accuracy:
  143. accuracy = self.accuracy(training_data, convert=True)
  144. training_accuracy.append(accuracy)
  145. print "Accuracy on training data: {} / {}".format(
  146. accuracy, n)
  147. if monitor_evaluation_cost:
  148. cost = self.total_cost(evaluation_data, lmbda, convert=True)
  149. evaluation_cost.append(cost)
  150. print "Cost on evaluation data: {}".format(cost)
  151. if monitor_evaluation_accuracy:
  152. accuracy = self.accuracy(evaluation_data)
  153. evaluation_accuracy.append(accuracy)
  154. print "Accuracy on evaluation data: {} / {}".format(
  155. self.accuracy(evaluation_data), n_data)
  156. print
  157. return evaluation_cost, evaluation_accuracy, \
  158. training_cost, training_accuracy
  159. def update_mini_batch(self, mini_batch, eta, lmbda, n):
  160. """Update the network's weights and biases by applying gradient
  161. descent using backpropagation to a single mini batch. The
  162. ``mini_batch`` is a list of tuples ``(x, y)``, ``eta`` is the
  163. learning rate, ``lmbda`` is the regularization parameter, and
  164. ``n`` is the total size of the training data set.
  165. """
  166. nabla_b = [np.zeros(b.shape) for b in self.biases]
  167. nabla_w = [np.zeros(w.shape) for w in self.weights]
  168. for x, y in mini_batch:
  169. delta_nabla_b, delta_nabla_w = self.backprop(x, y)
  170. nabla_b = [nb+dnb for nb, dnb in zip(nabla_b, delta_nabla_b)]
  171. nabla_w = [nw+dnw for nw, dnw in zip(nabla_w, delta_nabla_w)]
  172. self.weights = [(1-eta*(lmbda/n))*w-(eta/len(mini_batch))*nw
  173. for w, nw in zip(self.weights, nabla_w)]
  174. self.biases = [b-(eta/len(mini_batch))*nb
  175. for b, nb in zip(self.biases, nabla_b)]
  176. def backprop(self, x, y):
  177. """Return a tuple ``(nabla_b, nabla_w)`` representing the
  178. gradient for the cost function C_x. ``nabla_b`` and
  179. ``nabla_w`` are layer-by-layer lists of numpy arrays, similar
  180. to ``self.biases`` and ``self.weights``."""
  181. nabla_b = [np.zeros(b.shape) for b in self.biases]
  182. nabla_w = [np.zeros(w.shape) for w in self.weights]
  183. # feedforward
  184. activation = x
  185. activations = [x] # list to store all the activations, layer by layer
  186. zs = [] # list to store all the z vectors, layer by layer
  187. for b, w in zip(self.biases, self.weights):
  188. z = np.dot(w, activation)+b
  189. zs.append(z)
  190. activation = sigmoid(z)
  191. activations.append(activation)
  192. # backward pass
  193. delta = (self.cost).delta(zs[-1], activations[-1], y)
  194. nabla_b[-1] = delta
  195. nabla_w[-1] = np.dot(delta, activations[-2].transpose())
  196. # Note that the variable l in the loop below is used a little
  197. # differently to the notation in Chapter 2 of the book. Here,
  198. # l = 1 means the last layer of neurons, l = 2 is the
  199. # second-last layer, and so on. It's a renumbering of the
  200. # scheme in the book, used here to take advantage of the fact
  201. # that Python can use negative indices in lists.
  202. for l in xrange(2, self.num_layers):
  203. z = zs[-l]
  204. sp = sigmoid_prime(z)
  205. delta = np.dot(self.weights[-l+1].transpose(), delta) * sp
  206. nabla_b[-l] = delta
  207. nabla_w[-l] = np.dot(delta, activations[-l-1].transpose())
  208. return (nabla_b, nabla_w)
  209. def accuracy(self, data, convert=False):
  210. """Return the number of inputs in ``data`` for which the neural
  211. network outputs the correct result. The neural network's
  212. output is assumed to be the index of whichever neuron in the
  213. final layer has the highest activation.
  214. The flag ``convert`` should be set to False if the data set is
  215. validation or test data (the usual case), and to True if the
  216. data set is the training data. The need for this flag arises
  217. due to differences in the way the results ``y`` are
  218. represented in the different data sets. In particular, it
  219. flags whether we need to convert between the different
  220. representations. It may seem strange to use different
  221. representations for the different data sets. Why not use the
  222. same representation for all three data sets? It's done for
  223. efficiency reasons -- the program usually evaluates the cost
  224. on the training data and the accuracy on other data sets.
  225. These are different types of computations, and using different
  226. representations speeds things up. More details on the
  227. representations can be found in
  228. mnist_loader.load_data_wrapper.
  229. """
  230. if convert:
  231. results = [(np.argmax(self.feedforward(x)), np.argmax(y))
  232. for (x, y) in data]
  233. else:
  234. results = [(np.argmax(self.feedforward(x)), y)
  235. for (x, y) in data]
  236. return sum(int(x == y) for (x, y) in results)
  237. def total_cost(self, data, lmbda, convert=False):
  238. """Return the total cost for the data set ``data``. The flag
  239. ``convert`` should be set to False if the data set is the
  240. training data (the usual case), and to True if the data set is
  241. the validation or test data. See comments on the similar (but
  242. reversed) convention for the ``accuracy`` method, above.
  243. """
  244. cost = 0.0
  245. for x, y in data:
  246. a = self.feedforward(x)
  247. if convert: y = vectorized_result(y)
  248. cost += self.cost.fn(a, y)/len(data)
  249. cost += 0.5*(lmbda/len(data))*sum(
  250. np.linalg.norm(w)**2 for w in self.weights)
  251. return cost
  252. def save(self, filename):
  253. """Save the neural network to the file ``filename``."""
  254. data = {"sizes": self.sizes,
  255. "weights": [w.tolist() for w in self.weights],
  256. "biases": [b.tolist() for b in self.biases],
  257. "cost": str(self.cost.__name__)}
  258. f = open(filename, "w")
  259. json.dump(data, f)
  260. f.close()
  261. #### Loading a Network
  262. def load(filename):
  263. """Load a neural network from the file ``filename``. Returns an
  264. instance of Network.
  265. """
  266. f = open(filename, "r")
  267. data = json.load(f)
  268. f.close()
  269. cost = getattr(sys.modules[__name__], data["cost"])
  270. net = Network(data["sizes"], cost=cost)
  271. net.weights = [np.array(w) for w in data["weights"]]
  272. net.biases = [np.array(b) for b in data["biases"]]
  273. return net
  274. #### Miscellaneous functions
  275. def vectorized_result(j):
  276. """Return a 10-dimensional unit vector with a 1.0 in the j'th position
  277. and zeroes elsewhere. This is used to convert a digit (0...9)
  278. into a corresponding desired output from the neural network.
  279. """
  280. e = np.zeros((10, 1))
  281. e[j] = 1.0
  282. return e
  283. def sigmoid(z):
  284. """The sigmoid function."""
  285. return 1.0/(1.0+np.exp(-z))
  286. def sigmoid_prime(z):
  287. """Derivative of the sigmoid function."""
  288. return sigmoid(z)*(1-sigmoid(z))
添加新批注
在作者公开此批注前,只有你和作者可见。
回复批注