PRML演習問題 4.6(基本) - 機械学習基礎理論独習

問題

$(4.27)$ で与えられるクラス問共分散行列と $(4.28)$ で与えられるクラス内共分散行列のそれぞれの定義と、
$(4.34),(4.36)$ および $4.1.5$ 節で述べた目的値を使って、
二乗和誤差関数を最小化する表現 $(4.33)$ が $(4.37)$ の形で件けることを示せ。

参考

$\begin{eqnarray} {\bf S}_B=({\bf m}_2-{\bf m}_1)({\bf m}_2-{\bf m}_1)^\top\tag{4.27} \end{eqnarray}$

$\begin{eqnarray} {\bf S}_{\rm W}=\sum_{n\in{\mathcal C}_1}({\bf x}_n-{\bf m}_1)({\bf x}_n-{\bf m}_1)^\top+\sum_{n\in{\mathcal C}_2}({\bf x}_n-{\bf m}_2)({\bf x}_n-{\bf m}_2)^\top\tag{4.28} \end{eqnarray}$

$\begin{eqnarray} \sum_{n=1}^N({\bf w}^\top{\bf x}_n+w_0+t_n){\bf x}_n=0\tag{4.33} \end{eqnarray}$

$\begin{eqnarray} w_0=-{\bf w}^\top{\bf m}\tag{4.34} \end{eqnarray}$

$\begin{eqnarray} {\bf m}=\frac{1}{N}\sum_{n=1}^N{\bf x}_n=\frac{1}{N}(N_1{\bf m}_1+N_2{\bf m}_2)\tag{4.36} \end{eqnarray}$

$\begin{eqnarray} \left({\bf S}_W+\frac{N_1N_2}{N}{\bf S}_B\right){\bf w}=N({\bf m}_1-{\bf m}_2)\tag{4.37} \end{eqnarray}$

解答

先に $\displaystyle\sum_{i\in{\mathcal C}_k}({\bf x}_i-{\bf m}_k)({\bf x}_i-{\bf m}_k)^\top$ を計算します。(後で使います)

$\begin{eqnarray} \sum_{i\in{\mathcal C}_k}({\bf x}_i-{\bf m}_k)({\bf x}_i-{\bf m}_k)^\top&=&\sum_{i\in{\mathcal C}_k}({\bf x}_i{\bf x}_i^\top-{\bf x}_i{\bf m}_k^\top-{\bf m}_k{\bf x}_i^\top+{\bf m}_k{\bf m}_k^\top)\\ &=&\sum_{i\in{\mathcal C}_k}{\bf x}_i{\bf x}_i^\top-N_k{\bf m}_k{\bf m}_k^\top\tag{1} \end{eqnarray}$

$(4.33)$ の左辺を変形します。

$\begin{eqnarray} &&\sum_{n=1}^N({\bf w}^\top{\bf x}_n+w_0-t_n){\bf x}_n\\ &=&\sum_{n=1}^N({\bf w}^\top{\bf x}_n\underbrace{-{\bf w}^\top{\bf m}}_{(4.34)}-t_n){\bf x}_n\\ &=&\sum_{n=1}^N({\bf w}^\top{\bf x}_n{\bf x}_n-{\bf w}^\top{\bf m}{\bf x}_n-{\bf x}_nt_n)\\ &=&\sum_{n=1}^N({\bf x}_n{\bf x}_n^\top{\bf w}-{\bf x}_n{\bf m}^\top{\bf w}-{\bf x}_nt_n)\\ &=&\sum_{n=1}^N(({\bf x}_n{\bf x}_n^\top-{\bf x}_n{\bf m}^\top){\bf w}-{\bf x}_nt_n)\\ &=&\sum_{n\in{\mathcal C}_1}(({\bf x}_n{\bf x}_n^\top-{\bf x}_n{\bf m}^\top){\bf w}-{\bf x}_nt_n)+\sum_{m\in{\mathcal C}_2}(({\bf x}_m{\bf x}_m^\top-{\bf x}_m{\bf m}^\top){\bf w}-{\bf x}_mt_m)\\ &=&\left(\sum_{n\in{\mathcal C}_1}{\bf x}_n{\bf x}_n^\top-N_1{\bf m}_1{\bf m}_1^\top\right){\bf w}-N_1{\bf m}_1\underbrace{\frac{N}{N_1}}_{\mathcal{C}_1}+\left(\sum_{m\in{\mathcal C}_2}{\bf x}_m{\bf x}_m^\top-N_2{\bf m}_2{\bf m}_2^\top\right){\bf w}+N_2{\bf m}_2\underbrace{\frac{N}{N_2}}_{\mathcal{C}_2}\\ &=&\left(\sum_{n\in{\mathcal C}_1}{\bf x}_n{\bf x}_n^\top+\sum_{m\in{\mathcal C}_2}{\bf x}_m{\bf x}_m^\top-(N_1{\bf m}_1+N_2{\bf m}_2){\bf m}^\top\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left(\underbrace{\sum_{n\in{\mathcal C}_1}({\bf x}_n-{\bf m}_1)({\bf x}_n-{\bf m}_1)^\top+N_1{\bf m}_1{\bf m}_1^\top}_{(1)}+\underbrace{\sum_{m\in{\mathcal C}_2}({\bf x}_m-{\bf m}_2)({\bf x}_m-{\bf m}_2)^\top+N_2{\bf m}_2{\bf m}_2^\top}_{(1)}-(N_1{\bf m}_1+N_2{\bf m}_2){\bf m}^\top\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left(\underbrace{{\bf S}_W}_{(4.28)}+N_1{\bf m}_1{\bf m}_1^\top+N_2{\bf m}_2{\bf m}_2^\top-(N_1{\bf m}_1+N_2{\bf m}_2)\underbrace{\frac{1}{N}(N_1{\bf m}_1^\top+N_2{\bf m}_2^\top)}_{(4.36)}\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left({\bf S}_W+\left(N_1-\frac{N_1^2}{N}\right){\bf m}_1{\bf m}_1^\top-\frac{N_1N_2}{N}({\bf m}_1{\bf m}_2^\top+{\bf m}_2{\bf m}_1^\top)+\left(N_2-\frac{N_2^2}{N}\right){\bf m}_2{\bf m}_2^\top\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left({\bf S}_W+\frac{(N_1+N_2)N_1-N_1^2}{N}{\bf m}_1{\bf m}_1^\top-\frac{N_1N_2}{N}({\bf m}_1{\bf m}_2^\top+{\bf m}_2{\bf m}_1^\top)+\frac{(N_1+N_2)N_2-N_2^2}{N}{\bf m}_2{\bf m}_2^\top\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left({\bf S}_W+\frac{N_2N_1}{N}({\bf m}_1{\bf m}_1^\top-{\bf m}_1{\bf m}_2^\top-{\bf m}_2{\bf m}_1^\top+{\bf m}_2{\bf m}_2^\top)\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left({\bf S}_W+\frac{N_2N_1}{N}({\bf m}_2-{\bf m}_1)({\bf m}_2-{\bf m}_1)^\top\right){\bf w}-N({\bf m}_1-{\bf m}_2)\\ &=&\left({\bf S}_W+\frac{N_2N_1}{N}\underbrace{{\bf S}_B}_{(4.27)}\right){\bf w}-N({\bf m}_1-{\bf m}_2)\tag{2} \end{eqnarray}$

$(4.33)$ に $(2)$ を代入します。

$\begin{eqnarray} \left({\bf S}_W+\frac{N_2N_1}{N}{\bf S}_B\right){\bf w}-N({\bf m}_1-{\bf m}_2)=0\\ \Leftrightarrow\left({\bf S}_W+\frac{N_1N_2}{N}{\bf S}_B\right){\bf w}=N({\bf m}_1-{\bf m}_2)\tag{3} \end{eqnarray}$

以上より、題意が示せました。