Krylov Subspace

$$ \mathbf{Ax} = \mathbf{b} \\ \mathbf{r} = \mathbf{b} - \mathbf{A} \hat{\mathbf{x}} $$

image.png

$$ \mathbf{x} = \hat{\mathbf{x}}^{(k)} + \mathbf{A}^{-1} \mathbf{r}^{(k)} \\ \hat{\mathbf{x}}^{(k+1)} = \hat{\mathbf{x}}^{(k)} + \alpha^{(k)} \mathbf{r}^{(k)} \\ \mathbf{r}^{(k+1)} = \mathbf{b} - \mathbf{A} \hat{\mathbf{x}}^{(k+1)} $$

image.png

$$ \text{Iteration 0:} \\ \hat{\mathbf{x}}^{(0)} = \mathbf{0} \\ \mathbf{r}^{(0)} = \mathbf{b} \\[0.3cm]

\text{Iteration 1:} \\ \hat{\mathbf{x}}^{(1)} = \alpha^{(0)} \mathbf{b} \\ \mathbf{r}^{(1)} = \mathbf{b} - \alpha^{(0)} \mathbf{A} \mathbf{b} \\[0.3cm]

\text{Iteration 2:} \\ \hat{\mathbf{x}}^{(2)} = (\alpha^{(0)} + \alpha^{(1)}) \mathbf{b} - \alpha^{(0)} \alpha^{(1)} \mathbf{A} \mathbf{b} \\ \mathbf{r}^{(2)} = \mathbf{b} - (\alpha^{(0)} + \alpha^{(1)}) \mathbf{A} \mathbf{b} + \alpha^{(0)} \alpha^{(1)} \mathbf{A}^2 \mathbf{b} \\[0.3cm]

\text{Iteration } k \text{:} \\ \hat{\mathbf{x}}^{(k)} = \beta_0^{(k)} \mathbf{b} + \beta_1^{(k)} \mathbf{A} \mathbf{b} + \dots + \beta_{k-1}^{(k)} \mathbf{A}^{k-1} \mathbf{b} \\ \mathbf{r}^{(k)} = \mathbf{b} - \beta_0^{(k)} \mathbf{A} \mathbf{b} - \beta_1^{(k)} \mathbf{A}^2 \mathbf{b} - \dots - \beta_{k-1}^{(k)} \mathbf{A}^k \mathbf{b} $$

$$ \boxed{\hat{\mathbf{x}}^{(k)} \in \kappa_k(\mathbf{A}, \mathbf{b}) = \text{span}(\mathbf{b}, \mathbf{A} \mathbf{b}, \mathbf{A}^2 \mathbf{b}, \dots, \mathbf{A}^{k-1} \mathbf{b})} $$

Arnoldi’s Iteration

$$ \mathbf{A} = \mathbf{Q} \mathbf{H} \mathbf{Q}^* \\ \mathbf{A} \mathbf{Q} = \mathbf{Q} \mathbf{H} $$

$$ \mathbf{A} \mathbf{Q}n = \mathbf{Q}{n+1} \tilde{\mathbf{H}}_n \\ \mathbf{A} \in \mathbb{C}^{m \times m}, \mathbf{Q}_n \in \mathbb{C}^{m \times n}, \tilde{\mathbf{H}}n \in \mathbb{C}^{(n+1) \times n} \\ \mathbf{A} \mathbf{q}n = h{1,n} \mathbf{q}1 + \dots + h{n,n} \mathbf{q}n + h{n+1,n} \mathbf{q}{n+1} $$

$$ \text{Iteration 0:} \\ \mathbf{q}_1 = \frac{\mathbf{b}}{\|\mathbf{b}\|} \\ \mathbf{q}_1 \in \text{span}(\mathbf{b}) \\[0.3cm]

\text{Iteration 1:} \\ \mathbf{A} \mathbf{q}1 = h{1,1} \mathbf{q}1 + h{2,1} \mathbf{q}_2 \\ \mathbf{q}_2 = \frac{\mathbf{A} \mathbf{q}1 - h{1,1} \mathbf{q}1}{h{2,1}} \\ \mathbf{q}_2 \in \text{span}(\mathbf{b}, \mathbf{A} \mathbf{b}), \mathbf{q}_2 \perp \mathbf{q}_1 \\[0.3cm]

\text{Iteration 2:} \\ \mathbf{A} \mathbf{q}2 = h{1,2} \mathbf{q}1 + h{2,2} \mathbf{q}2 + h{3,2} \mathbf{q}_3 \\ \mathbf{q}_3 = \frac{\mathbf{A} \mathbf{q}2 - h{1,2} \mathbf{q}1 - h{2,2} \mathbf{q}2}{h{3,2}} \\ \mathbf{q}_3 \in \text{span}(\mathbf{b}, \mathbf{A} \mathbf{b}, \mathbf{A}^2 \mathbf{b}), \mathbf{q}_3 \perp \{\mathbf{q}_1, \mathbf{q}_2\} \\[0.3cm]

\text{Iteration } n \text{:} \\ \mathbf{A} \mathbf{q}n = h{1,n} \mathbf{q}1 + \dots + h{n,n} \mathbf{q}n + h{n+1,n} \mathbf{q}{n+1} \\ \mathbf{q}{n+1} = \frac{\mathbf{A} \mathbf{q}n - h{1,n} \mathbf{q}1 - \dots - h{n,n} \mathbf{q}n}{h{n+1,n}} \\ \mathbf{q}{n+1} \in \text{span}(\mathbf{b}, \mathbf{A} \mathbf{b}, \dots, \mathbf{A}^n \mathbf{b}), \mathbf{q}{n+1} \perp \{\mathbf{q}_1, \dots, \mathbf{q}_n\} $$

$$ \boxed{\text{span}(\mathbf{b}, \mathbf{A} \mathbf{b}, \dots, \mathbf{A}^n \mathbf{b}) \mapsto \text{span}(\mathbf{q}_1, \mathbf{q}2, \dots, \mathbf{q}{n+1})} $$

Generalised Minimal Residual Method (GMRES)

$$ \mathbf{A} \mathbf{x} = \mathbf{b} \\ \mathbf{A} \mathbf{Q}n \mathbf{y} = \mathbf{b} \\ \mathbf{Q}{n+1} \tilde{\mathbf{H}}_n \mathbf{y} = \mathbf{b} \\ \tilde{\mathbf{H}}n \mathbf{y} = \mathbf{Q}{n+1}^* \mathbf{b} $$

$$ \boxed{ \begin{array}{c} \tilde{\mathbf{H}}_n \mathbf{y} = \|\mathbf{b}\| \mathbf{e}_1 \\ \mathbf{x} = \mathbf{Q}_n \mathbf{y} \\ \mathbf{x} \in \mathbb{C}^m, \mathbf{y} \in \mathbb{C}^n, n \leq m \end{array} } $$