Thanks to R Parr, C Guesterin
!
" ! #
$ $ %& !
'
$(
'
)
( * ++ '
, * - !
#.
/ - - 0 0
1
, * , 0
- + % % % %
/ '2 )3'221
" -
$
5 6 % 5
6 7
...
+
! - !
! * -
5 % % % % 6 (1
(1 ) (1
(1 ) = 3(1
(1 )2
2 8 !" ,
4
$: (# '
- + " ,
-
( *;- < = -
/ > -? !+ , ! 1
: =
- - , - +
9
2 ;( *< A*
2 + + =
h t
ln[ (1 ) ] = [h ln + t ln (1 ) ] = +
h t t
ht t
==
+
(1) +
(1 )
h t t e!!!
g
+ =0 = v er a
(1 ) t +h j us
ta
So @
0 , ; <1
H
MLE =
H + T
)+ % $('6 C9 6 D.
@
)+ D D % $('6 D.
@
B
3 *& !
No
mr
al
(
& !
=0
.6
; <
, =
0.
%
04
8)
$ 6 C F
& !
G C F N
or
m
al
(
% =0
.6
6D.
@ 6D.D4E ,
=0
. 00
D % D 48
)
6D.
@ 6D.DD4E
E
" -
P( D | ) for fixed =0.6
n=5 n=50
5 - 6D.
@! * #C % ,
H
5 - - !
P( D | ) for fixed D
5 - 6 ! * %
D
,, *> '8
1 m
" , Sm = Xi - + + * + .
+. MD% N
m i =1
2 O F K P
5I 7 Q
5IJ2 G J < K G 7 C
- - .
.. ? .
..
2 + * # - ! +
L !
2 -
* ,, *> 8
L, 6 F
H
2 + * 6MLE =
H + T
( R -
OD
5A ( *
5A 5 - - A : !
# 0 - !
# %
0 6 D.%
0 - - 7 6 D.
H9
8 L, O C C 2 460.2
/ - # 0 *1
2 # 0 #
- !
;! < 9D79D
! " # "$
* %
- -> + - + ,
4
Two (related) Distributions:
Parameter, Instances
Uniform density
1.0
= 0.1 = 0.5 = 0.8
T T H
T T H
T H T
T T H
H H H
T H H
9
Two (related) Distributions:
Parameter, Instances
Uniform density
1.0 1.0 1.0
= 0.1 = 0.5 = 0.8
T T H
T T H
T H T
T T H
H H H
T H H
@
( *
#
3
8+ 0 * : 5J"
B
( *, - !
#
(# , !
/ - 1
: # 0 *
2 ,
? *
7, ,
% #" & "'
!( # ) ! "E
- G 5
(# , !
)+ ST %-
$ CF-
3 , %-O G C F-G
& ! - C F- F-G
H
5 - ,
Prior P() Likelihood P(D|)
D
So Posterior is same form as Prior!! Conjugate!
5 " -
5 T %
" %
5 -
J T F % F
Prior + observe 1 head + observe
27 more heads;
18 tails
? * 5
)+
5 T %
" 0
- #
5 -
J T F % F
5 ! 5J "'
!( # , !
, * * %
! - 0
5J> , 0 , >
3 * 5
5 -
, ! 0 ,
* *
': ! +
- * , !
$A5 $ :
:
A - + % . #
, - $A5
H + H 1
MAP = arg max P( | D) =
H + H + T + T 2
U (# '6 * :5"J
$(
- , ; - + *< 7 %7 : ,
$A5 #
f (MAP )
4
$A5, -
$A5 #
H + H 1
MAP = arg max P( | D) =
H + H + T + T 2
8+ : - !
#,
A % ;, * <
% +& "9
5 ! ,
0
5 T %
- + %
/ - - : F , 1
1
P( Xm+1 = H | D) = P( Xm+1 = H | , D) P( | D) d
0
1
= Beta( : H + mH , T + mT ) d
0
H + mH
= EBeta ( : H + mH ,T + mT ) [] =
H + mH + T + mT @
A + ;' ! *<
%- %
0
6 F-
,, !+ =
6 C F-
'*
% 6 % D.
9
D% D 6 D% D.
9
B% 6 D% D.B
B
A !- +
8+ =
Fix m, change
8+ : ,
A % ;, * <
#! &* +&
"
Fix , change m
,!- " +
5 = - %.%
> 8+ =
E
* 2 *
H +m H 6 F 6 F
E[] =
H + mH + T + m T 8+ =
H mH
= + $(
' *
m + m +
H mH m
= +
m + m m +
$('
U $(' %- 0
6D
m
$(
'
H
*, $
/ , + #7 - !
#111
? ' - !
# ? +
(# , ! , ! "
5S 6 6 6 .
.#
6 D
/ "'
!( ,
0 - % , * %
" ! F % % # F #
i + mi
) " P ( X m +1 = i | D ) =
j
( j + m j )
D
!
" ! #
$ $ %& !
'
$(
'
)
5 ,)
( *5 ,)
$ + " -
A
!"- " ) %
0 V+ !
5" - - - , !
2 5 ,)
A,, ,
*- ! * !
ST %
W6 SF- WT F-%
2 ,)
ST S% S
WT W% W
X6 SFW XT SFW% SF W
$&)6 $ & )
6) + + -
$ + )
A 7 ) , -
+! 6 I % K
12,1 22,1
!+ ! : =
1, 2 2, 2
2 2
0 %? 6 'I : G :? G ? K
!+ !
!
%
; + 7 , < : : : D
4
0.25
0.2
0.15
0.1
0.05
3
2
3
1 2
0 1
1 0
1
2
2
3 3
1 0
=
0 1
9
3
3
3 2 1 0 1 2 3
1 0
=
0 1 @
2 )
B
0.25
0.2
0.15
0.1
0.05
0
3
1 2
1
2 0
1
3 2
3
1 0.5
=
0.5 1 E
3
3
3 2 1 0 1 2 3
1 0.5
=
0.5 1 H
0.25
0.2
0.15
0.1
0.05
0
3
1 2
1
2 0
1
3 2
3
1 0 .8
=
0.8 1 4D
3
3
3 2 1 0 1 2 3
1 0 .8
=
0.8 1 4
& -
4
$ + ) ':
4
': ,)
6 6 " * % # )
6 " *%
Marginal
44
3 , 5 ,)
2 , ! , 8 - - .
.
, D% !+ ! )
* )
'+ * )
) *
,, ,
49
3 , 5 ,)
A) - !
!, !-
+! ,
!+ ! :
8 !
8
- ? - + -
+ - 8 !
4@
3 , 5 ,)
$ * ,) )
)+ x = ( xa , xb ), = ( a , b )
aa ab
=
ba bb
$ * " -
p( xa ) = N ( xa | a , aa )
$ * =- * *
4B
3 , 5 ,) &
,) )
aa
1
ab
= =
ba bb
" -
1
p ( xa | xb ) = N ( xa | a|b , aa )
a|b = a aa 1 ab ( xb a )
4E
& = *$ * = Y
*
4H
3 , 5 ,) &
A,, , ,) + -
)
2 : )
6A:F- )
3
- W, - :
: , - +*
9D
3 , 5 ,)
( , *! * - -
: 0 - )
(
, "+ - 0 ,
+ ! 0 0 )
-
( , 70 + * *
? , * ) - ,
*
9
( * ) 99
75
82
! , %"
93
, 7+ ... ! :
.
*.% : !
(
$ %
& !%
9
$(
', )
5 -. , ... ! "6 M: % %: N
N N N ( xi ) 2
1
P ( D | , ) = P ( xi | , ) = e 2 2
i =1 2 i =1
( *7 # ,
9
$(
', , )
/ $( MLE
T , 1
N
d ( xi ) 2 1 N
1 N
= = 2( xi ) = xi N
i =1 d 2 2
2 2 i =1 2
i =1
d N
ln P( D | , ) = 0 xi N = 0
d i =1
1 N
MLE = xi Just empirical mean!!
N i =1 94
$(
', & !
A* % + + =
N 2( xi ) 2
=
i 2 3
1
=0 2
MLE = ( xi ) 2
N i
Just empirical variance!! 99
MLE 1 N
- MLE = xi
N i =1
' , - ,, 'IK 6
- + M:% %: N
0 ! -
0 ! : K 6
'I
1 N
1 N
1 N
E[ MLE ] = E xi = E[ xi ] = =
N i =1 N i =1 N i =1
9@
( *)
$(
'
$(
', ) + ! )
': ! , Z
3- + !
! !
#1
or k
ew
om
H
9B
/ 1
0 $ V$(
' 0 $
$ $
9E
' * $ + )
)+ M1 % % 1 N% $(
'
1
MLE = xi
N i
1
MLE = ( xi ) ( xi )T
N i
9H
* ,
)
? *
$ )
& ! / " -
5 ,
P( |,)
@D
$A5, ,)
P ( | D , , , )
P( D | , ) P( | , )
d d d
ln P( D | ) P( ) = ln P( D | ) + ln P( )
d d d
( xi ) ( )
= 2
i 2
xi
+ 2
i
2
... = 0 MAP =
N 1
+
2
2 @
$A5, ,)
xi
+ 2
i
2
MAP =
N 1
+
2
2
,# 0 *%
$A5 $('Z
, P %
$A5 /') A&' A)' ,
$( ' ; <
@
( ,)
)
* #
! * : +
" , * ,) -
+ ! ,
0 ! +
$ : % # - % !
.
2 *, ) *
@
$: ,)
/ : -
1
" 0 * !K - ,)
p( x) = k N ( x | k , k )
k =1
K
k =1
k =1
! ! !) %
, )
- - : +0 * )
@4
$: ,) ':
@9
/ # 0
5 - - D
5 '
$(
'
,, * 8 5A
*
%" ! -
) %
$A5
@@
@B
!
- 6- 1
ln =
- 6 F -
1
ln (1 ) =
(1 )
@E
!! ! , + -
A ! ! * : % %: S % %S
- - 5S%W 6 5S 5WJS
5SJW 6 5WJS 5S C 5W
5S % %S 6
5S 5S JS 5S#JS % %S#7 5S JS % %S 7
@H
- +
> 8
S0 , %+ !
Var ( X )
P(| X E ( X ) | c) 2
c
& ! *+ ! ! , *
BD
+* ! ,2 $
A - +
> 8
Var ( X )
P(| X E ( X ) | c)
c2
Xi 1 Var ( X )
Var ( X ) = Var = i 2 Var ( X i ) =
i
n n n
Var ( X )
lim n P(| X E ( X ) | c) lim n 2
= 0
nc
B
& -
'+ ! ! G0 # - -
A* % ) %
+ - , = -
) 6A , + M ,) 6 AN
5 , + %S
& S 6 - + , + S
! ! * ! 6 J& S J 5S6: 6
! : S6: : 6
B
$ + ) ':
