Expectation

Linearity

$$\begin{array}{rl}\mathbb{E}[aX+b]& =\sum _x(ax+b)\cdot p(x)\\ & =\sum _x(ax\cdot p(x)+b\cdot p(x))\\ & =\sum _{x}ax\cdot p(x)+\sum _{x}b\cdot p(x)\\ & =a\sum _{x}x\cdot p(x)+b\sum _{x}p(x)\\ & =a\mathbb{E}[X]+b\end{array}$$

Variance

$Var(X)=\mathbb{E}[X^2]+\mathbb{E}[X{]}^2$

$$\begin{array}{rl}Var(X)& =\mathbb{E}[(X-\mu {)}^2]\\ & =\sum _x(x-\mu {)}^{2}p(x)\\ & =\sum _x(x^2-2x\mu -{\mu }^2)p(x)\\ & =\sum _x(x^2\cdot p(x)-2x\mu \cdot p(x)+{\mu }^2\cdot p(x))\\ & =\sum _x{x}^2\cdot p(x)-\sum _{x}2x\mu \cdot p(x)+\sum _x{\mu }^2\cdot p(x)\\ & =\sum _x{x}^2\cdot p(x)-2\mu \sum _{x}x\cdot p(x)+{\mu }^2\sum _x\cdot p(x)\\ & =\mathbb{E}[X^2]-2\mu \mathbb{E}[X]+{\mu }^2\times 1\\ & =\mathbb{E}[X^2]-2\mu \cdot \mu +{\mu }^2\\ & =\mathbb{E}[X^2]-{\mu }^2\\ & =\mathbb{E}[X^2]-\mathbb{E}[X{]}^2\end{array}$$

Binomial Distribution

expectation

$$\begin{array}{rl}\mathbb{E}[X]& =\sum _{k=0}^{n}k\cdot (\genfrac{}{}{0ex}{}{n}{k})p^k(1-p{)}^{n-k}\\ & =\sum _{k=1}^{n}k\cdot (\genfrac{}{}{0ex}{}{n}{k})p^k(1-p{)}^{n-k}\\ & =\sum _{k=1}^{n}k\cdot \frac{n!}{k!(n-k)!}\cdot p^k(1-p{)}^{n-k}\\ & =\sum _{k=1}^n\frac{n!}{(k-1)!(n-k)!}\cdot p^k(1-p{)}^{n-k}\\ & =\sum _{k=1}^n\frac{n\cdot (n-1)!}{(k-1)!(n-1-(k-1))!}\cdot p^k(1-p{)}^{n-k}\\ & =n\sum _{k=1}^n(\genfrac{}{}{0ex}{}{n-1}{k-1})p^k(1-p{)}^{n-k}\\ & =np\sum _{k=1}^n(\genfrac{}{}{0ex}{}{n-1}{k-1})p^{k-1}(1-p{)}^{n-k}\\ & =np\sum _{k=1}^n(\genfrac{}{}{0ex}{}{n-1}{k-1})p^{k-1}(1-p{)}^{(n-1)-(k-1)}\\ & \text{Let }m=n-1\text{, }j=k-1:\\ & =np\sum _{j=0}^m(\genfrac{}{}{0ex}{}{m}{j})p^j(1-p{)}^{m-j}\\ & \sum _{j=0}^m(\genfrac{}{}{0ex}{}{m}{j})p^j(1-p{)}^{m-j}\text{ is the sum of all probabilities of Bin(m,p),}\\ & =np\times 1\\ & =np\end{array}$$

variance

$$\begin{array}{rl}Var(X)& =\mathbb{E}[X^2]-\mathbb{E}[X{]}^2\\ & =\mathbb{E}[X^2]-(np{)}^2\end{array}$$

The proccess is similiar to above.

$$\begin{array}{rl}\mathbb{E}[X^2]& =\sum _{k=0}^n{k}^2\cdot (\genfrac{}{}{0ex}{}{n}{k})p^k(1-p{)}^{n-k}\\ & =\sum _{k=1}^n{k}^2\cdot (\genfrac{}{}{0ex}{}{n}{k})p^k(1-p{)}^{n-k}\\ & =n\sum _{k=1}^{n}k\cdot (\genfrac{}{}{0ex}{}{n-1}{k-1})p^k(1-p{)}^{n-k}\\ & =np\sum _{k=1}^{n}k\cdot (\genfrac{}{}{0ex}{}{n-1}{k-1})p^{k-1}(1-p{)}^{n-1-(k-1)}\\ & \text{Let }m=n-1\text{, }j=k-1:\\ & =np\sum _{j=0}^m(j+1)\cdot (\genfrac{}{}{0ex}{}{m}{j})p^j(1-p{)}^{m-j}\\ & =np(\sum _{j=0}^{m}j\cdot (\genfrac{}{}{0ex}{}{m}{j})p^j(1-p{)}^{m-j}+\sum _{j=0}^m(\genfrac{}{}{0ex}{}{m}{j})p^j(1-p{)}^{m-j})\\ & \sum _{j=0}^{m}j\cdot (\genfrac{}{}{0ex}{}{m}{j})p^j(1-p{)}^{m-j}\text{ is the expectation evaluated above}\\ & =np(mp+1)\\ & =np(np-p+1)\\ & =(np{)}^2-n{p}^2+np\\ & =(np{)}^2+np(1-p)\end{array}$$$$\begin{array}{rl}\mathbb{E}[X^2]-(np{)}^2& =(np{)}^2+np(1-p)-(np{)}^2\\ & =np(1-p)\end{array}$$

Geometric Distribution

expectation

$$\begin{array}{rl}\mathbb{E}[X]& =\sum _{k=1}^{\mathrm{\infty }}k\cdot (1-p{)}^{k-1}p\\ & =p\sum _{k=1}^{\mathrm{\infty }}k\cdot (1-p{)}^{k-1}\\ & \text{Let }q=1-p\text{,}\\ & =p\sum _{k=1}^{\mathrm{\infty }}k\cdot q^{k-1}\\ & =\frac{p}{q}\sum _{k=1}^{\mathrm{\infty }}k\cdot q^k\\ & \text{This is the arithmetico-geometric sequence.}\\ & \text{Apply sum formula,}\\ & =\frac{p}{q}\cdot \frac{q}{(1-q{)}^2}\\ & =\frac{p}{(1-(1-p){)}^2}\\ & =\frac{p}{p^2}\\ & =\frac{1}{p}\end{array}$$

variance

$$\begin{array}{rl}Var(X)& =\mathbb{E}[X^2]-\mathbb{E}[X{]}^2\\ & =\mathbb{E}[X^2]-\frac{1}{p^2}\end{array}$$$$\begin{array}{rl}\mathbb{E}[X^2]& =\sum _{k=1}^{\mathrm{\infty }}{k}^2\cdot (1-p{)}^{k-1}p\\ & =p\sum _{k=1}^{\mathrm{\infty }}{k}^2\cdot (1-p{)}^{k-1}\\ & \text{Let }q=1-p\text{,}\\ & =p\sum _{k=1}^{\mathrm{\infty }}{k}^2\cdot q^{k-1}\end{array}$$

See Differentiation of Geometric Series.

$$\begin{array}{rl}p\sum _{k=1}^{\mathrm{\infty }}{k}^2\cdot q^{k-1}& =\frac{p}{q}\sum _{k=1}^{\mathrm{\infty }}{k}^2\cdot q^k\\ & =\frac{p}{q}\cdot \frac{q(1+q)}{(1-q{)}^3}\\ & =\frac{p(1+q)}{(1-q{)}^3}\\ & =\frac{p(1+1-p)}{(1-1+p{)}^3}\\ & =\frac{2p-p^2}{p^3}\\ & =\frac{2-p}{p^2}\end{array}$$$$\begin{array}{rl}Var(X)& =\mathbb{E}[X^2]-\mathbb{E}[X{]}^2\\ & =\frac{2-p}{p^2}-\frac{1}{p^2}\\ & =\frac{1-p}{p^2}\end{array}$$

Poisson Distribution

probability mass function

Derived from binomial distribution,

$$\begin{array}{rl}& \underset{n\to \mathrm{\infty }}{lim}(\genfrac{}{}{0ex}{}{n}{k}){\left(\frac{\lambda }{n}\right)}^k{(1-\frac{\lambda }{n})}^{n-k}\\ =& \underset{n\to \mathrm{\infty }}{lim}\frac{n^k}{k!}\cdot \frac{{\lambda }^k}{n^k}{(1-\frac{\lambda }{n})}^n\cdot {(1-\frac{\lambda }{n})}^{-k}\\ =& \frac{{\lambda }^k}{k!}\cdot e^{-\lambda }\cdot 1\\ =& e^{-\lambda }\frac{{\lambda }^k}{k!}\end{array}$$

expectation

$$\begin{array}{rl}\mathbb{E}[X]& =\sum _{k=0}^{\mathrm{\infty }}k\cdot e^{-\lambda }\frac{{\lambda }^k}{k!}\\ & =e^{-\lambda }\sum _{k=0}^{\mathrm{\infty }}k\cdot \frac{{\lambda }^k}{k!}\\ & =e^{-\lambda }\sum _{k=1}^{\mathrm{\infty }}k\cdot \frac{{\lambda }^k}{k!}\\ & =e^{-\lambda }\sum _{k=1}^{\mathrm{\infty }}\frac{{\lambda }^k}{(k-1)!}\\ & =\lambda e^{-\lambda }\sum _{k=1}^{\mathrm{\infty }}\frac{{\lambda }^{k-1}}{(k-1)!}\\ & \text{Let }j=k-1\text{,}\\ & =\lambda e^{-\lambda }\sum _{j=0}^{\mathrm{\infty }}\frac{{\lambda }^j}{j!}\\ & \text{By the definition }{e}^x=\sum _{n=0}^{\mathrm{\infty }}\frac{x^n}{n!}\text{,}\\ & =\lambda e^{-\lambda }\cdot e^{\lambda }\\ & =\lambda \end{array}$$

variance

$$\begin{array}{rl}Var(X)& =\mathbb{E}[X^2]-\mathbb{E}[X{]}^2\\ & =\mathbb{E}[X^2]-{\lambda }^2\end{array}$$$$\begin{array}{rl}\mathbb{E}[X^2]& =\sum _{k=0}^{\mathrm{\infty }}{k}^2\cdot e^{-\lambda }\frac{{\lambda }^k}{k!}\\ & =e^{-\lambda }\sum _{k=0}^{\mathrm{\infty }}{k}^2\frac{{\lambda }^k}{k!}\\ & =e^{-\lambda }\sum _{k=0}^{\mathrm{\infty }}(k(k-1)+k)\frac{{\lambda }^k}{k!}\\ & =e^{-\lambda }(\sum _{k=0}^{\mathrm{\infty }}k(k-1)\frac{{\lambda }^k}{k!}+\sum _{k=0}^{\mathrm{\infty }}k\frac{{\lambda }^k}{k!})\\ & \sum _{k=0}^{\mathrm{\infty }}k\frac{{\lambda }^k}{k!}\text{ is evaluated above.}\end{array}$$$$\begin{array}{rl}\sum _{k=0}^{\mathrm{\infty }}k(k-1)\frac{{\lambda }^k}{k!}& =\sum _{k=2}^{\mathrm{\infty }}k(k-1)\frac{{\lambda }^k}{k!}\\ & =\sum _{k=2}^{\mathrm{\infty }}\frac{{\lambda }^{k-2}\cdot {\lambda }^2}{(k-2)!}\\ & ={\lambda }^2\sum _{k=2}^{\mathrm{\infty }}\frac{{\lambda }^{k-2}}{(k-2)!}\\ & \text{Let }j=k-2\\ & ={\lambda }^2\sum _{j=0}^{\mathrm{\infty }}\frac{{\lambda }^j}{j!}\\ & ={\lambda }^2{e}^{\lambda }\end{array}$$$$\begin{array}{rl}& e^{-\lambda }(\sum _{k=0}^{\mathrm{\infty }}k(k-1)\frac{{\lambda }^k}{k!}+\sum _{k=0}^{\mathrm{\infty }}k\frac{{\lambda }^k}{k!})\\ =& e^{-\lambda }({\lambda }^2{e}^{\lambda }+\lambda e^{\lambda })\\ =& e^{-\lambda }\cdot e^{\lambda }({\lambda }^2+\lambda )\\ =& {\lambda }^2+\lambda \end{array}$$$$\begin{array}{rl}\mathbb{E}[X^2]-{\lambda }^2& ={\lambda }^2+\lambda -{\lambda }^2\\ & =\lambda \end{array}$$