update

doc/src/week3/programs/#simplenn.py#

@@ -0,0 +1,65 @@
+# import necessary packages
+import numpy as np
+import matplotlib.pyplot as plt
+
+def sigmoid(z):
+    return 1.0/(1.0+np.exp(-z))
+
+def forwardpropagation(x):
+    # weighted sum of inputs to the hidden layer
+    z_1 = np.matmul(x, w_1) + b_1
+    # activation in the hidden layer
+    a_1 = sigmoid(z_1)
+    # weighted sum of inputs to the output layer
+    z_2 = np.matmul(a_1, w_2) + b_2
+    a_2 = z_2
+    return a_1, a_2
+
+def backpropagation(x, y):
+    a_1, a_2 = forwardpropagation(x)
+    # parameter delta for the output layer, note that a_2=z_2 and its derivative wrt z_2 is just 1
+    delta_2 = a_2 - y
+    print(0.5*((a_2-y)**2))
+    # delta for  the hidden layer
+    delta_1 = np.matmul(delta_2, w_2.T) * a_1 * (1 - a_1)
+    # gradients for the output layer
+    output_weights_gradient = np.matmul(a_1.T, delta_2)
+    output_bias_gradient = np.sum(delta_2, axis=0)
+    # gradient for the hidden layer
+    hidden_weights_gradient = np.matmul(x.T, delta_1)
+    hidden_bias_gradient = np.sum(delta_1, axis=0)
+    return output_weights_gradient, output_bias_gradient, hidden_weights_gradient, hidden_bias_gradient
+
+
+# ensure the same random numbers appear every time
+np.random.seed(0)
+# Input variable
+x = np.array([4.0],dtype=np.float64)
+# Target values
+y = 2*x+1.0 
+
+# Defining the neural network, only scalars
+n_inputs, n_features = X.shape
+#n_features = 2
+n_hidden_neurons = 2
+n_outputs = 1
+
+# Initialize the network
+# weights and bias in the hidden layer
+w_1 = np.random.randn(n_features, n_hidden_neurons)
+b_1 = np.zeros(n_hidden_neurons) + 0.01
+
+# weights and bias in the output layer
+w_2 = np.random.randn(n_hidden_neurons, n_outputs)
+b_2 = np.zeros(n_outputs) + 0.01
+
+eta = 0.1
+for i in range(100):
+    # calculate gradients
+    derivW2, derivB2, derivW1, derivB1 = backpropagation(x, y)
+    # update weights and biases
+    w_2 -= eta * derivW2
+    b_2 -= eta * derivB2
+    w_1 -= eta * derivW1
+    b_1 -= eta * derivB1
+

doc/src/week3/programs/.#simplenn.py

@@ -0,0 +1 @@
+mhjensen@Mortens-MacBook-Pro.local.59471

doc/src/week4/week4.do.txt

@@ -389,7 +389,7 @@ of the learning rates. Feel free to add either hyperparameters with an
 $l_1$ norm or an $l_2$ norm and discuss your results.
 Discuss your results as functions of the amount of training data and various learning rates.
 
-_Challenge:_ Try to cnage activation functions and replace the hard-coded analytical expressions with automatic derivation via either _autograd_ or _JAX_.
+_Challenge:_ Try to change the activation functions and replace the hard-coded analytical expressions with automatic derivation via either _autograd_ or _JAX_.
 
 !split
 ===== Simple neural network and the  back propagation equations  =====
@@ -624,7 +624,23 @@ where $\eta$ is the learning rate.
 !split
 ===== Program example  =====
 
+We extend our simple code to a function which depends on two variable $x_0$ and $x_1$, that is
+!bt
+\[
+y=f(x_0,x_1)=x_0^2+3x_0x_1+x_1^2+5.
+\]
+!et
+We feed our network with $n=100$ entries $x_0$ and $x_1$. We have thus two features represented by these variable and an input matrix/design matrix $\bm{X}\in \mathbf{R}^{n\times 2}$
+!bt
+\[
+\bm{X}=\begin{bmatrix} x_{00} & x_{01} \\ x_{00} & x_{01} \\ x_{10} & x_{11} \\ x_{20} & x_{21} \\ \dots & \dots \\ \dots & \dots \\ x_{n-20} & x_{n-21} \\ x_{n-10} & x_{n-11} \end{bmatrix}.
+\]
+!et
+
 
+!bc pycod
+
+!ec
 
 !split
 ===== Getting serious, the  back propagation equations for a neural network =====
@@ -2733,14 +2749,14 @@ Computing polynomial products can be implemented efficiently if we rewrite the m
 We note first that the new coefficients are given as
 
 !bt
-\begin{split}
+\begin{align}
 \delta_0=&\alpha_0\beta_0\\
 \delta_1=&\alpha_1\beta_0+\alpha_1\beta_0\\
 \delta_2=&\alpha_0\beta_2+\alpha_1\beta_1+\alpha_2\beta_0\\
 \delta_3=&\alpha_1\beta_2+\alpha_2\beta_1+\alpha_0\beta_3\\
 \delta_4=&\alpha_2\beta_2+\alpha_1\beta_3\\
 \delta_5=&\alpha_2\beta_3.\\
-\end{split}
+\end{align}
 !et