exercise10.tex

\documentclass[11pt]{article}
% % \def\hidesolutions{}
\input{header}
\begin{document}
\exsheet{10}{21}{November} % parameters are the number of the session and the day




\begin{exercise}
    \begin{align*}
        ReLU(x) 
        &= \begin{cases} 0 & \text{ if } x < 0 \\ x & \text{ if } 0 \leq x \end{cases}
        \\
        ReLU^{k}(x) 
        &= \begin{cases} 0 & \text{ if } x < 0 \\ x^{k} & \text{ if } 0 \leq x \end{cases} \text{ where } k \in \mathbb N_0
        \\
        f(x) 
        &= 
        \begin{cases} 
            -x-2            & \text{ if } -2 < x < -1
            \\
            - x^{2} - 2x - 1 & \text{ if } -1 < x < 0 
            \\ 
              x^{2} - 2x + 1 & \text{ if } 0 <  x < 1
            \\
            2-x            & \text{ if }  1 < x < 2
            \\
            0              & \text{ otherwise }  
        \end{cases}
        \\
        j(x) 
        &= 
        \begin{cases} 
            x      & \text{ if } x < 0
            \\
            1 + x  & \text{ if } 0 \leq x             
        \end{cases}
        .
    \end{align*}
    Here, employ the definition of distributional derivatives and check against the formula for piecewise differentiable functions as obtained during lecture. 
\end{exercise}
\begin{solution}
    \begin{itemize}
        \item For the $ReLU$ function, we have for any $\phi \in \mathcal D(\mathbb{R})$,
        \begin{align*}
            \langle ReLU', \phi \rangle = -\langle ReLU, \phi' \rangle = -\int_{-\infty}^{\infty} ReLU(x) \phi'(x) dx = -\int_{0}^{\infty} x \phi'(x) dx = \int_{0}^{\infty} \phi(x) dx = \langle H, \phi \rangle,
        \end{align*}
        where $H$ is the Heaviside step function. Thus, we conclude that $ReLU' = H$ in the sense of distributions.
        \item For $k > 1$, we find for any $\phi \in \mathcal D(\mathbb{R})$,
        \begin{align*}
            \langle (ReLU^k)', \phi \rangle &= - \langle ReLU^k, \phi' \rangle = - \int_{-\infty}^{\infty} ReLU^k(x) \phi'(x) dx \\
            &= - \int_{0}^{\infty} x^k \phi'(x) dx = \int_{0}^{\infty} k x^{k-1} \phi(x) dx = \langle k ReLU^{k-1}, \phi \rangle.
        \end{align*}
        Therefore, we conclude that $(ReLU^k)' = k ReLU^{k-1}$ in the sense of distributions.
        \item For the function $f$, we have for any $\phi \in \mathcal D(\mathbb{R})$,
        \begin{align*}
            \langle f', \phi \rangle 
			= 
			- \langle f, \phi'\rangle 
			&
			= 
			\int_{-2}^{-1}(x + 2) \phi'(x) dx 
			\\&\qquad 
			+ \int_{-1}^{0} (x^2 + 2x + 1) \phi'(x) dx
			\\&\qquad 
			- \int_{0}^{1} (x^2 - 2x + 1) \phi'(x) dx 
			\\&\qquad 
			+ \int_{1}^{2} (x - 2) \phi'(x) dx 
			\\
            &
			= \phi(-1) - \int_{-2}^{-1} \phi(x) dx 
			\\&\qquad 
			+ \phi(0) - \int_{-1}^{0} 2(x + 1) \phi(x)dx
			\\&\qquad 
			+ \phi(0) + \int_{0}^{1} 2(x - 1) \phi(x) dx 
			\\&\qquad 
			+ \phi(1) - \int_{1}^{2} \phi(x) dx 
			\\
            &
			= \int_{-2}^{-1} (-1) \phi(x) dx + \int_{-1}^0 (-2(x + 1)) \phi(x) dx 
			\\
            &\qquad 
			+ \int_{0}^{1} 2(x - 1) \phi(x) dx + \int_{1}^{2} (-1) \phi(x) dx + \phi(-1) + 2\phi(0) + \phi(1)
			\\
            &
			= \langle g + \delta_{-1} + 2\delta_{0} + \delta_{1} , \phi \rangle
			,
        \end{align*}
        where the function $g: \mathbb{R} \to \mathbb{R}$ is defined as
        \begin{align*}
            g(x) = \begin{cases} 
					-1 & \text{ if } -2 < x \leq -1 
					\\ 
					-2(x + 1) & \text{ if } -1 < x \leq 0 
					\\ 
					2(x - 1) & \text{ if } 0 < x \leq 1 
					\\ 
					-1 & \text{ if } 1 < x \leq 2 
					\\ 
					0 & \text{ otherwise } 
					\end{cases}.
        \end{align*}
        Hence, we conclude that $f' = g + \delta_{-1} + 2\delta_{0} + \delta_{1}$. Note that this corresponds exactly to the formula found during the lecture.
		\item For the function $j$, we have for any $\phi \in \mathcal D(\mathbb{R})$,
        \begin{align*}
            \langle j', \phi \rangle 
			= 
			- \langle j, \phi'\rangle 
			&
			= 
			- 
			\int_{-\infty}^{0} x \phi'(x) dx 
			- 
			\int_{0}^{\infty} ( 1 + x ) \phi'(x) dx 
			\\&\qquad 
			= 
			\int_{-\infty}^{0} \phi(x) dx 
			+
			\int_{0}^{\infty}  \phi(x) dx + \phi(0) 
			\\&\qquad 
			= 
			\int_{-\infty}^{\infty} \phi(x) dx + \phi(0) 
			.
		\end{align*}
		We conclude that $j' = 1 + \delta_0$.
		Note that we can also write $j(x) = x + H(x)$, where $H$ is the heaviside function.
    \end{itemize}
\end{solution}




\begin{exercise}
    Let $f : \bbR \rightarrow \bbR$ be a function with period $1$ and 
    \begin{align*}
        f(x) = x, \qquad 0 < x < 1.
    \end{align*}
    Find the distributional derivative. 
\end{exercise}
\begin{solution}
    For a test function $\phi \in \mathcal{D}(\mathbb{R})$, we have
    \begin{align*}
        \langle f', \phi \rangle 
        &= 
        - \langle f, \phi' \rangle 
        = 
        -\sum_{n \in \mathbb{Z}} \int_n^{n+1} (x - n) \phi'(x) dx 
        ,
    \end{align*}
    Here, we have used that the function looks like $x-n$ over each interval $(n,n-1)$:
    over the interval from $n$ to $n+1$, it looks like a linear function with slope $1$ that equals $0$ at $x=n$.
    Now, use integration by parts:
    \begin{align*}
        -\sum_{n \in \mathbb{Z}} \int_n^{n+1} (x - n) \phi'(x) dx 
        &
        = 
        -\sum_{n \in \mathbb{Z}} \left[ \phi(n + 1) - \int_{n}^{n+1} \phi(x) dx \right]
        \\&
        =
        -
        \sum_{n \in \mathbb{Z}} \phi(n + 1)
        +
        \sum_{n \in \mathbb{Z}} \int_{n}^{n+1} \phi(x) dx
        \\&
        =
        -
        \sum_{n \in \mathbb{Z}} \phi(n)
        +
        \sum_{n \in \mathbb{Z}} \int_{n}^{n+1} \phi(x) dx
        \\&
        =
        -
        \sum_{n \in \mathbb{Z}} \phi(n)
        +
        \int_{-\infty}^{\infty} \phi(x) dx
        .
    \end{align*}
    On a technical note, we notice that the sum only involves a finite number of terms and that the integral is finite, too, because $\phi$ is smooth with compact support. 
    Hence, we have the distributional derivative equals the constant function $1$ minus the Dirac comb.
    \begin{align*}
        f' = 1 - \Delta_1.
    \end{align*}
    We observe that this accounts for the fact that $f$ is a piecewise linear function with slope equal $1$ on the intervals and with jumps $-1$ at each of the interval boundaries.
\end{solution}






\begin{exercise}
    Compute the Fourier coefficients of the function $f$ that has period $T = 2\pi$ and satisfies
    \[
        f(x) = \left\{\begin{array}{ll} 1 & 0 \leq x < \pi \\ 0 & \pi < x \leq 2\pi \end{array}\right.
    \]
\end{exercise}
\begin{solution}     
    We notice that the period $T$ equals $T = 2\pi$. 
    We have defined the first coefficient to be just the average:
    \begin{align*}
        \frac{a_0 }{2}
        &= 
        \frac{1}{2\pi} \int_0^{2\pi} f(x) \, dx 
        = 
        \frac{1}{2\pi} \int_0^\pi \, dx  = \frac 1 2
        .
    \end{align*}
    For the other coefficients, we use the formulas from the lecture:
    \begin{align*}
        a_n 
        &= 
        \frac{2}{2\pi} \int_0^{2\pi} f(x) \cos\left(\frac{2\pi n x}{T}\right) \, dx 
        \\&= 
        \frac{2}{2\pi} \int_0^\pi \cos\left(\frac{2\pi n x}{T}\right) \, dx 
        \\
        &= 
        \frac{2}{2\pi} \left[ \frac{T}{2\pi n} \sin\left(\frac{2\pi n x}{T}\right) \right]_{x=0}^{x=\pi}
        \\&= 
        \frac{2}{2\pi} \frac{T}{2\pi n} \left( \sin\left(\frac{2\pi n \pi}{T}\right) - \sin(0) \right) 
        \\&= 
        \frac{2}{2\pi} \frac{2\pi}{2\pi n} \left( \sin\left(n \pi \right) - \sin(0) \right) 
        \\&= 
        0
    \end{align*}
    because $\sin$ equals zero at integer multiples of $\pi$.
    For the sine mode coefficients, we calculate similarly:
    \begin{align*}
        b_n 
        &= 
        \frac{2}{2\pi}
        \int_0^{2\pi} f(x) \sin(nx) \, dx 
        \\
        &= 
        \frac{1}{\pi}
        \int_0^\pi \sin(nx) \, dx 
        \\
        &= 
        \frac{1}{\pi}
        \left[-\frac{\cos(nx)}{n} \right]_{x=0}^{x=\pi} 
        \\
        &= 
        -
        \frac{1}{\pi}
        \frac{1}{n} \left[ \cos(n\pi) - \cos(0) \right] 
        \\
        &= 
        -
        \frac{1}{n\pi}
        \left[ (-1)^n - 1 \right] 
        \\
        &= 
        \frac{1}{n\pi}
        \begin{cases}
            0 & \text{if } n \text{ is even} 
            \\
            2 & \text{if } n \text{ is odd}
        \end{cases}
    \end{align*}
    Thus we find 
    \begin{align}
        \frac{a_0}{2} = 0.5,
        \qquad 
        a_1 = a_2 = \dots = 0,
    \end{align}
    and either $b_n = 0$ if $n$ is even and $b_n = \frac{2}{n\pi}$ if $n$ is odd. 
\end{solution}


\begin{exercise}
    Compute the Fourier coefficients of the following functions, which have period $T = 1$ and have the given values over the interval $[0,1)$:
    \begin{itemize}
     \item 
     \[
        f(x) = x^{2}
     \]
     \item 
     \[
        g(x) = (1-x)x
     \]
     \item 
     \[
        h(x) = |\sin( 2 \pi x )|
     \]
     %
    \end{itemize}
\end{exercise}
\begin{solution}     
    \begin{itemize}
        \item 
        To compute the Fourier coefficients of $f(x) = x^2$ over the interval $[0,1)$ with a period of $1$, use the formulas from the lecture:
        \[
            \frac{a_0}{2} = \int_{0}^{1} f(x) dx  = \int_{0}^{1} x^2 dx  = \left[ \frac{x^3}{3} \right]_{0}^{1} = \frac 1 3.
        \]
        For other coefficients, we apply integration by parts twice, and we use that $\sin$ vanishes at multiples of $\pi$:
        \begin{align*}
            a_n 
            &
            = 
            \frac{2}{T}
            \int_{0}^{1} x^2 \cos(2\pi n x) dx
            \\&
            =
            \frac{2}{T}
            \left[ x^2 \frac{ \sin(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            -
            \frac{2}{T}
            \int_{0}^{1} \frac{ 2x \sin(2\pi n x) }{ 2\pi n } dx
            \\&
            =
            \frac{2}{T}
            \left[ x^2 \frac{ \sin(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            -
            \frac{4}{T}
            \int_{0}^{1} x \frac{ \sin(2\pi n x) }{ 2\pi n } dx
            \\&
            =
            \frac{2}{T}
            \left[ x^2 \frac{ \sin(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            -
            \frac{4}{T}
            \left[ x \frac{ -\cos(2\pi n x) }{ 4\pi^2 n^2 } \right]_{x=0}^{x=1}
            +
            \frac{4}{T}
            \int_{0}^{1} \frac{ -\cos(2\pi n x) }{ 4\pi^2 n^2 } dx
            \\&
            =
            \frac{2}{T}
            \left[ x^2 \frac{ \sin(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            -
            \frac{4}{T}
            \left[ x \frac{ -\cos(2\pi n x) }{ 4\pi^2 n^2 } \right]_{x=0}^{x=1}
            +
            \frac{4}{T}
            \left[ \frac{ -\sin(2\pi n x) }{ 8\pi^3 n^3 } \right]_{x=0}^{x=1}
            \\&
            = 
            \frac{1}{\pi^2 n^2}
        \end{align*}
        \begin{align*}
            b_n 
            &
            = 
            \frac{2}{T}
            \int_{0}^{1} x^2 \sin(2\pi n x) dx
            \\&
            =
            \frac{2}{T}
            \left[- x^2 \frac{ \cos(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{2}{T}
            \int_{0}^{1} \frac{ 2x \cos(2\pi n x) }{ 2\pi n } dx
            \\&
            =
            \frac{2}{T}
            \left[- x^2 \frac{ \cos(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{4}{T}
            \int_{0}^{1} \frac{ x \cos(2\pi n x) }{ 2\pi n } dx
            \\&
            =
            \frac{2}{T}
            \left[ -x^2 \frac{ \cos(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{4}{T}
            \left[ x \frac{ \sin(2\pi n x) }{ 4\pi^2 n^2 } \right]_{x=0}^{x=1}
            -
            \frac{4}{T}
            \int_{0}^{1} \frac{ \sin(2\pi n x) }{ 4\pi^2 n^2 } dx
            \\&
            =
            \frac{2}{T}
            \left[ -x^2 \frac{ \cos(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{4}{T}
            \left[ x \frac{ \sin(2\pi n x) }{ 4\pi^2 n^2 } \right]_{x=0}^{x=1}
            +
            \frac{4}{T}
            \left[ \frac{ \cos(2\pi n x) }{ 8\pi^3 n^3 } \right]_{x=0}^{x=1}
            \\&
            = 
            -\frac{2}{T}\frac{1}{2\pi n} + \frac{4}{T}\frac{1}{8\pi^3n^3} -  \frac{4}{T}\frac{1}{8\pi^3n^3} = -\frac{1}{\pi n}
        \end{align*}
        \item 
        We notice that the function $g(x) = (1-x)x$ can be written $g(x) = x - f(x)$ over the interval $[0,1)$. 
        In other words, $g$ is the difference of the sawtooth function (which equals $x$ over $[0,1)$ and repeats with period $T=1$),
        and the periodic function $f(x)$ seen earlier.
        
        We will only need to determine the Fourier coefficients of $\tilde{g}(x) = x$ since we already computed the Fourier coefficients of $f(x)$. 
        We calculate the average 
        \[
            \frac{a_0^{\tilde g}}{2} = \int_{0}^{1} \tilde{g}(x) dx  = \int_{0}^{1} x dx  = \left[ \frac{x^2}{2} \right]_{0}^{1} = \frac 1 2.
        \]
        We calculate 
        \begin{align*}
            a_n^{\tilde g} 
            &
            = 
            \frac{2}{T}
            \int_{0}^{1} x \cos(2\pi n x) dx
            \\&
            =
            \frac{2}{T}
            \left[ x \frac{ \sin(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            -
            \frac{2}{T}
            \int_{0}^{1} \frac{ \sin(2\pi n x) }{ 2\pi n } dx
            \\&
            =
            \frac{2}{T}
            \left[ x \frac{ \sin(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{2}{T}
            \left[ \frac{ \cos(2\pi n x) }{ 4\pi^2 n^2 } \right]_{x=0}^{x=1}
            \\&
            =
            \frac{2}{T}
            \frac{1}{2\pi n}
            \left( \sin(2\pi n) - \sin(0) \right)
            +
            \frac{2}{T}
            \frac{1}{ 4\pi^2 n^2 }
            \left( \cos(2\pi n) - \cos(0) \right)
            \\&
            =
            0
        \end{align*}	
        Lastly, 
        \begin{align*}
            b_n^{\tilde g} 
            &
            = 
            \frac{2}{T}
            \int_{0}^{1} x \sin(2\pi n x) dx
            \\&
            =
            \frac{2}{T}
            \left[- x \frac{ \cos(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{2}{T}
            \int_{0}^{1} \frac{ \cos(2\pi n x) }{ 2\pi n } dx
            \\&
            =
           \frac{2}{T}
            \left[- x \frac{ \cos(2\pi n x) }{ 2\pi n } \right]_{x=0}^{x=1}
            +
            \frac{2}{T}
            \left[ \frac{ \sin(2\pi n x) }{ 4\pi^2 n^2 } \right]_{x=0}^{x=1}
            \\&
            = 
            -\frac{2}{T}\frac{1}{2\pi n} =  -\frac{1}{\pi n} 
        \end{align*}	
        Now the Fourier coefficients of $g(x)$ are given by
                \begin{align*}
                    &b_n = -\frac{1}{\pi n} - - \frac{1}{\pi n} = 0
                    \\& 
            a_n = 0 - \frac{1}{\pi^2 n^2} =  - \frac{1}{\pi^2 n^2} 
                    \\&
            \frac{a_0}{2} = \frac{1}{2} - \frac{1}{3} = \frac{1}{6}
        \end{align*}	
        \item
        Note that $h(x)$ is an even function.
        We first check that the sine modes are have coefficients equal zero.
        \begin{align*}
            b_n 
            &
            = 
            \frac{2}{T}
            \int_{0}^{T} |\sin( 2\pi x)| \sin(2\pi n x) dx
            \\&
            =
            2
            \int_{0}^{1} |\sin( 2\pi x)| \sin(2\pi n x) dx
            \\&
            =
            2
            \int_{0}^{\frac 1 2} \sin( 2\pi x) \sin(2\pi n x) dx
            -
            2
            \int_{\frac 1 2}^{1} \sin( 2\pi x) \sin(2\pi n x) dx
            .            
        \end{align*}
        One possibility to proceed is via substitution in the second integral, setting $x = y + \frac 1 2$. 
        We then find 
        \begin{align*}
            &
            \int_{\frac 1 2}^{1} 
            \sin( 2\pi x) \sin(2\pi n x) dx
            \\&\qquad 
            =
            \int_{0}^{\frac 1 2} 
            \sin( 2\pi (x+ 0.5)) \sin(2\pi n (x+ 0.5)) dx
            \\&\qquad 
            =
            \int_{0}^{\frac 1 2} 
            \sin( 2\pi x + \pi ) \sin(2\pi n x + \pi n) dx
            =
            (-1)^{n+1}
            \int_{0}^{\frac 1 2} 
            \sin( 2\pi x ) \sin(2\pi n x + \pi n) dx
            .
        \end{align*}
        In the last step, we have used the symmetry properties of the sine function. 
        Hence 
        \begin{align*}
            b_n 
            &
            =
            2 \left( 1 - (-1)^{n+1} \right)
            \int_{0}^{\frac 1 2} \sin( 2\pi x) \sin(2\pi n x) dx
            .
        \end{align*}
        If $n$ is odd, then the last expression is already zero. 
        Otherwise, if $n$ is even, we want to show that last integral is zero. 
        We use the sum formula:
        \begin{align*}
            \sin( 2\pi x) \sin(2\pi n x)
            =
            \frac{1}{2}
            \cos( 2\pi (n-1) x)
            -
            \frac{1}{2}
            \cos( 2\pi (n+1) x)
            .
        \end{align*}
        With that observation, we find 
        \begin{align*}
            b_n 
            &
            =
            \frac{1}{2}
            \int_{0}^{\frac 1 2} 
            \cos( 2\pi (n-1) x)
            -
            \cos( 2\pi (n+1) x)
            \ dx
            \\&
            =
            \frac{1}{2}
            \left[ 
            \frac{ \sin( 2\pi (n-1) x) }{ 2\pi (n-1)}
            -
            \frac{ \sin( 2\pi (n+1) x) }{ 2\pi (n+1)}
            \right]_{x=0}^{x=\frac 1 2}
            .
        \end{align*}
        For $x=0$, the two sine terms are already zero, and for $x=\frac 1 2$, the two sine terms are evaluated at integer multiples of $\pi$ and hence equal zero, too. 
        We conclude that $b_n = 0$. 
        
        
        Next, we calculate the average 
        \[
            \frac{a_0}{2}
            = 
            \frac{1}{T} \int_{0}^{1} |\sin 2\pi x| dx  
            = 
            \frac{2}{T} \int_{0}^{\frac 1 2} \sin 2\pi x dx  
            =  
            \frac 2 T
            \left[ \frac{-1}{2\pi} \cos{2\pi x} \right]_{0}^{\frac 1 2} 
            =
            \frac{2}{\pi} 
            .
        \]
        The coefficients $a_n$ for the cosine modes with $n \geq 1$ are:
        \begin{align*}
            a_n 
            &
            = 
            \frac{4}{T}
            \int_{0}^{\frac 1 2} \sin 2\pi x \cos(2\pi n x) dx
            \\&
            =
            \frac{4}{T}
            \int_{0}^{\frac 1 2} \frac 1 2 \sin{2\pi(1-n)x} + \frac 1 2 \sin{2\pi(n+1)x} dx
            \\&
		=
            \frac{4}{T}
            \int_{0}^{\frac 1 2} -\frac 1 2 \sin{2\pi(n-1)x} + \frac 1 2 \sin{2\pi(n+1)x} dx
            \\&
            =
            \left[\frac{ \cos2\pi(n-1)x }{ \pi(n-1) } \right]_{x=0}^{x=\frac 1 2}
            +
            \left[- \frac{ \cos2\pi(n+1)x }{ \pi(n+1) } \right]_{x=0}^{x=\frac 1 2}
            \\&
            = 
            \frac{(-1)^{n-1}}{\pi(n-1)} -  \frac{1}{\pi(n-1)}  - \frac{(-1)^{n+1}}{\pi(n+1)} + \frac{1}{\pi(n+1)}
            \\&
            = -\frac{1+(-1)^{n}}{\pi(n-1)} +  \frac{1-(-1)^{n+1}}{\pi(n+1)} 
            \\&
            = -\frac{1+(-1)^{n}}{\pi(n-1)} +  \frac{1+(-1)^{n}}{\pi(n+1)} 
        \end{align*}	
        Therefore, when $n \geq 1$:
        \[a_n = 
        \begin{cases}
        0 ,\quad &\text{if } n\text{ is odd,}\\
        -\frac{2}{\pi(n-1)} + \frac{2}{\pi(n+1)}  ,\quad &\text{if } n\text{ is even,}
        \end{cases}
        \]
       \end{itemize}
\end{solution}







\end{document}