The Secrets to Learning NLP by Lloyd Johnson Copy

download The Secrets to Learning NLP by Lloyd Johnson Copy

of 127

Transcript of The Secrets to Learning NLP by Lloyd Johnson Copy

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    1/127

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    2/127

    ! #$%&' ()*+ ), -.%/0&1-

    "# $%&'(%# )*(+ ,*% -.&/ 0**1 &/ -* /.(%2 13*4+25)2 (35 -*

    '(12 #*6% +&,2 02--2%7 8*32/-+#7 9- &/ :2%# '6;. ( /2;*35(%#

    )*(+ *, -.&/ -* 0**1 -* )232%(-2 (3# &3;*'27

    23 1)$ 4)$56* *6%7 4))' 3/)8 8+9

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    3/127

    !"#$%&'()*+ -$.%$ +/)*+ '/ '$00 &/1 ($%$ )2 3 0)$4500 +$*$%30)63')/*2 3%$ 0)$24 7)*8$ -$ (3#$ */

    803)9 /* '%1'( /% 3881%38&: -$ -)00 ;$ 0&)*+

    8/*2)2'$*'0& '(%/1+( '()2 2$9)*3%4

    /1% 2$9)*3%2 '(3' $#$%&'()*+ -$

    23& -)00 ;$ 3 0)$: 3*= /'($% '$38($%2 =/ */'4 A/2'

    /> '($9 ;$0)$#$ '($)% 0)$24 /1% 0)$2 -)00

    -/%C /1' %$300& -$00 )> &/1 38' 32 )> '($& 3%$ '%1$4!

    B F&;.(%5 G(35+2%@ H*BI*6352% *, ACD

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    4/127

    !"#$% '( )'*+%*+,

    !"#$%&'(#)%" +

    ,-.# )/ 0123 4

    5)/#%$6 %7 012 8

    ,-.# #% 19.$" :)$/# +;

    B.CC%$# +D

    F9#. F%&9G I+

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    5/127

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    6/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! # !

    !"#$%&'(#)%"

    $%&' ( )*+,- %&.+/ .012- 3&2+1!4*'52*,-*6

    7+15+.88*'5 9347: 8; *'*-*.< *8=2., -1 /*,8*,,

    *-? @%& =&1=%1 >&+& -..,'C- 2'-*< 1'& 3&> D&.+J, KG& -%.- ( -+2%.->., *' *- )1+ 8&? L+18 -%.- =1*'- )1+>.+/B

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    7/127

    !"#$% '#()*#)

    + , +

    !"#$ &' ()*+

    -!./ #0 -120#+!3)42*536 .0#407883)4/ 3* *26( 7 93%1

    :31"% #: *52%$ 5(75 35 67) ;1 (70% 5# %1:3)1 *383"" 5(75 01*2"5* :0#8 *52%$3)4 9(75

    87>1* 5(1 %3::101)61 ;15911) 1?61""1)61 7)% 7A10741=

    @(0#24( 5(1 1) %#9) 3)5# 5(011 6#01

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    8/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! # !

    !"#$% $%&'(%) (* (+% ,%$-*.) )/)(%0 1(+% 02,345

    (+$*.6+ 7+28+ *.$ %9:%$2%,8% 2) :$*8%))%3 -2' (+% ;2-%

    )%,)%) *; -2).'&5 '.32(*$/5 +%)%

    ;2-% )%,)%) '$% (+% .-51() 2,(* *.$ ,%$-*.) )/)(%0 ',3

    (+% )*.$8%) (+'( 7% $%&/ .:*, (* ?.2&3 *.$ 2,(%$,'&

    %9:%$2%,8%=

    &'()#*+', $%&'(%) (* &',6.'6% ',3 *(+%$ ,*,!-%$?'&8*00.,28'(2*, )/)(%0) (+$*.6+ 7+28+ *.$ ,%.$'&

    $%:$%)%,('(2*,) '$% 8*3%35 *$3%$%3 ',3 62-%, 0%',2,6=

    >+2) 2,8&.3%) :28(.$%)5 )*.,3)5 ;%%&2,6)5 (')(%)5 )0%&&)

    ',3 7*$3) 1)%&; ('&

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    9/127

    !"#$% '#()*#)

    + , +

    !"#$%&' %) *+,

    -!. /* 01/2"$ $#3)45 (16/)4 #)"$ 788) 8*917"/*(8% /) 9(8:;*? @(8 03)%1A8)91"* 9(19 1""#B8% 0#2 9(8 C2819/#)

    #0 -!.5 (#B86825 7841) "#)45 "#)4 780#28 9(19?

    D) :;,E5 F"028% G#2H$7*I/ B2#98 1 7##I C1""8% !"#$%"$

    '%( !'%#)* 9(19 %/*C3**8% 9(8 /%81 #0 !#4/C1" !868"*?G#2H$7*I/ *1/% *#A89(/)4 281""$ /)9828*9/)4 /) (/* 7##I?J8 *1/%5 KF"A#*9 1"" L*$C(#"#4/C1" L2#7"8A* B828 9(8

    28*3"9 #0 9(8 /)17/"/9$ #0 1 L82*#) 9# 921682*8 "#4/C1""868"*M? @(19>* /)9828*9/)4 78C13*8 (8 *3448*98% 9(19

    *#A89/A8 /) 9(8 039328 9(828 B#3"% 78 1 B(#""$

    L2#C8**8% %8*C2/L9/#) #0 9(8 (3A1) 78/)4 9(19 B#3"%C#A8 1"#)4 1)% 1C931""$ %8*C2/78 9(8 L2#C8**8* #0 1

    (3A1) 78/)4? @(19 /* B(19 B8 9#%1$ C1"" -832#+!/)43/*9/C .2#421AA/)4?

    N32/)4 9(8 :;

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    10/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! # !

    $%& '(& '()*+% '(&, (-. /&-0*' 1-*./&0 -*. 2$3&/)4

    0&-/)%&. '(-' '(&, (-. - %5&3)6)3 +)6' 76 87.&//)*+ 9

    :&)*+ -:/& '7 7:%&0;& ?(&, :&+-* '7 50-3')3& @&%'-/'

    ?(&0-5, )* '(& 3/-%%077=% -' '(& A*);&0%)', 76B-/)670*)- C-*'- B0$D -*. '(&)0

    @0)*.&0 )*)')-//, H$%' 7:%&0;&. '(&)0

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    11/127

    !"#$% '#()*#)

    + , +

    -.$/ 0.)%"12 .)% 324)%12 ".512 21).61% 715. 5# 8192#+

    !4):94*54; 5 9)54" 45 :#5 45* )1-

    ).61 5(.5 45 21.""$ *5.251% 5# ?1;#61 -4%1"$ @)#-)/

    A#""#-4): 5(1 ;(.):1 #B ).61 5# 8192#+!4):94*54;

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    12/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! # !

    $%&'() *$%+, -./ 0%,.- 1203$4 5+3 -.6- %$ 1.6- 789 %$

    6:2*-; $-*3)%+, />N 5 G22A 5:2*-S2&&*+%=6-%2+ 6+3 S.6+,/4

    O./$/ :22A$ =2+-6%+ -./ :6$%$ 2? -./ Q/-6 Q23/(; 6

    &23/( 2? =2&&*+%=6-%2+ -.6- 120A$ :) :/=2&%+, &20/

    $'/=%?%= 6+3 0/&2B%+, I/+/06(%$6-%2+$; T%$-20-%2+$ 6+3T/(/-%2+$ ?02& '/2'(/U$ (6+,*6,/ 6+3 -.2*,.-$4 O.%$120A 16$ '0%&60%() 3/0%B/3 ?02& -./%0 $-*3) 2? 9/0($

    6+3 P6-%04

    V2((21%+, -./$/ -12 :22A$ :2-. G6+3(/0 H I0%+3/0

    $-*3%/3 Q%(-2+ W0%=A$2+; 60,*6:() -./ :/$-

    .)'+2-./06'%$- 2? 2*0 -%&/; 6+3 ?02& -./%0 $-*3) 2? .%$

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    13/127

    !"#$% '#()*#)

    + , +

    -#./ 0(1$ 234"5*(1% 67001.)* #8 0(1 9$2)#05:

    ;1:()5 ;(1*1

    4##/* :#)075)1% 0(1 47*5* #8 0(1 =5"0#) =#%1"@ 7 B#%1"

    #8 :#BB3)5:705#) 0(70 -#./* 4$ 41:#B5)E B#.1 F7E31

    7)% 5)0.#%3:5)E E1)1.7"5*705#)*@ %5*0#.05#)* 7)%

    %1"105#)* 5)0# 21#2"1G* "7)E37E1 7)% 0(#3E(0*>

    C)01.1*05)E"$@ 0(1 =5"0#) =#%1" 5* 7"B#*0 0(1 :#B2"101

    #22#*501 #8 0(1 =107 =#%1" H 430 4#0( E10 .1"574"1

    .1*3"0*>

    I"05B701"$@ 0(1 =107 =#%1" D =5"0#) =#%1" *1.F1 0#

    2.#F1 J"8.1% K#.L$4*/5 :#..1:0 H 4$ 0.7F1.*5)E "#E5:7"

    "1F1"* 50 5* 2#**54"1 8#. 2.#4"1B* 0# M3*0 %5*72217.>

    ;(1*1 B#%1"* -1.1 M3*0 0(1 052 #8 0(1 5:141.E 0(#3E(>

    N.#B O7)%"1. D P.5)%1.G* *03%51* #8 0(1*1 0(.110(1.725*0*@ 7 -(#"1 851"% #8 5)8#.B705#) -7* :.1701%>

    O1$#)% 0(1*1 B#%1"* 0(1$ *07.01% %#:3B1)05)E )1-

    01:()5

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    14/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! # !

    $%&'( )*+ ,- .,&/0( '11/23/& ,4 1%5436,/- '00 %7/6 38/

    .%60& '- ' -/3 %9 3%%0- 9%6 26%&51,4: 6/0,';0/ 6/-503-#?@- '4& >##@- .,38 85:/ 2/6-%4'0,3,/-

    0,A/ B438%4( C%;;,4- 5-,4: ,3 3% 8/02 38%5-'4&- %9

    2/%20/ ='A/ 0'-3,4: 18'4:/- ,4 38/,6 0,7/-< B- %56

    '.'6/4/-- '- ' -%1,/3( ,416/'-/- '6%54& 38/

    ,=2%63'41/ %9 -/09 6/-2%4-,;,0,3( '4& 2/6-%4'0

    /=2%./6=/43D 38/-/ 3%%0- ;/1%=/ =%6/ '220,1';0/ '4&'11/--,;0/ /7/6( &'(

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    15/127

    !"#$% '#()*#)

    + ,- +

    !"#$ $& '(#)* +,)-$

    ./0( /)% 1213$ 410()5671 /)% 4(1#3$ 5) 8!9 (/*

    /::"50/;5"54$ 5) #73 1213$ %/$ "521*< =10/7*1 54 (/* ;11)

    >#%1""1% ?3#> *#>1 #? 4(1 >#*4 *7001**?7" 4(13/:5*4*

    4# 1213 "521@ 4(1 31*7"4* *:1/A ?#3 4(1>*1"21*< B(/4 $#7

    C#7"% ;1 "##A5)D 4# ?#07* #) C(1) ?53*4 "1/3)5)D 8173#+

    !5)D75*450 93#D3/>>5)D C5"" 2/3$ C5%1"$ ;/*1% #) $#73>#452/45#) ?#3 54< E1D/3%"1** #? C(14(13 $#7 :"/) 4# 7*1

    $#73 )1C *A5""* 5) / 4(13/:$@ ;7*5)1** #3 1%70/45#)/"

    0#)41F4 4(131 /31 *1213/" 0#)01:4* 4(/4 ?#3> 4(1

    ?#7)%/45#) $#7 C5"" )11%@ ;1?#31 $#7 0/) 1??104521"$ 7*1

    4(1 #4(13 8!9 4##"* /)% 410()5671*< G(1 ?#07* ?#3 4(5*

    ;##A 5* #) 4(1*1 *5F /31/*H

    ,< I1)*#3$ J0754$K< E/::#34L< J)0(#35)DM< N5"4#) N#%1"O< N14/ N#%1"P< N#%1""5)D

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    16/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! ## !

    $%&'()* +,-./* .' %''%&/.01 2() 3%,(4.&5 0 5((6

    ,(44-&.,0/() 0&6 2() 3%,(4.&5 5((6 0/ 789: ;/ .' /.11 01'(40F% &%0)1* 011 789 =)(,%''%' 4-,< %0'.%) 0&6 20'/%):

    +&,.11 5.A% *(- /

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    17/127

    !"#$% '#()*#)

    + ,- +

    .(/) $#0 12*3/4 3(/*/ *56 3/7()580/* $#0 95"" :/ 2

    "#); 92$ 3#924%* 0*5); /3/)3"$ 5) 3(/42>$

    2)% :0*5)/**? 2)% 5) $#04 /@/4$%2$ "5A/B C(/ "2);02;/

    >233/4)* #A 3(/ D5"3#) 2)% D/32 D#%/" 24/ /*>/752""$

    >#9/4A0"B

    E#4 $#04 :/)/A53? 9(23 A#""#9* 5* 5)A#41235#) #) /27( #A

    3(/*/ >4#7/**/*B .(23 5* 7#)325)/% 5) 3(5* *(#43 :##F 5*

    :$ )# 1/2)* 2"" 3(/4/ 5* 3# F)#9 #) /27( 3#>57? 5) A273/)354/ :##F* /65*3 #) /27( #A 3(/*/ 3/7()580/* 2"#)/B

    C(/ 5)A#41235#) 5* >4#@5%/% *# 3(23 $#0 95"" :/

    7#)A5%/)3 /)#0;( 3# 32F/ 2735#) )#9B C(/ >4273572"

    /6/475*/* 95"" ;5@/ $#0 3(/ #>>#430)53$ 3# 2>>"$ 3(/

    3/7()580/*? *# 3(23 :$ 3(/ /)% #A 3(5* :##F $#0 95""

    (2@/ 2 4/2""$ *#"5% 5%/2 #A (#9 3# 0*/ 9(23 $#0 (2@/

    "/24)/% 5) 2 >4273572" 92$ 5) $#04 #9) "5A/? (#9 3# 0*/ 533# 2AA/73 7(2);/ 5) #3(/4*G "5@/*? 2)% (#9 3# 7#)35)0/

    $#04 "/24)5); 5A %/*54/%B

    H# %# $#04*/"A 2 A2@#04I C4/23 3(5* 5)A#41235#) )/9"$?

    *32$ 7045#0*? 2)% ;5@/ $#04*/"A /@/4$ #>>#430)53$ 3#

    >427357/ 9(23 $#0 4/2%B

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    18/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    !"#$%&'(" *+ ,"%-./0 1#(&20

    %& '()*++,&-. ',+/(& 01,234(&. (&* (5 /6* 21*7/(14 (5

    89:. (;4*173* >,&?/* 267&-*4 51(>

    >(>*&/ /( >(>*&/@ A6(4* 267&-*4 2(>>?&,27/*

    >*7&,&- .5B(? 67(>*&/ /(>(>*&/@ H74*) (& /6* 267&-*4 /67/ B(? (;4*17++B 5(2?4 (& /6*4* 3*B =71/4 (5 7&(/6*1

    =*14(&G4 =6B4,(+(-BI

    ! C3,& 2(+(?1 JE6*/6*1 ,/ ,4 +,-6/*1 (1 )713*1K! C3,& /(&?4 JE6*/6*1 /6* /(&* (5 >?42+*4 ,4

    4B>>*/1,27+ (1 &(/ 4B>>*/1,27+K! H1*7/6,&- 4=**) JE6*/6*1 ;1*7/6,&- ,4 574/*1 (1

    4+(E*1K! H1*7/6,&- +(27/,(& JE6*/6*1 /6*BL1* ;1*7/6,&-

    51(> 6,-6*1 (1 +(E*1 ,& /6*,1 26*4/K

    ! 9(E*1 +,= 4,M* JE6*/6*1 ,/ 674 >(1* +,&*4 (1 +*44+,&*4K! 0B* 5(2?4 JE6*/6*1 /6*,1 *B*4 71* >(1*

    5(2?4*) (1 )*5(2?4*)K! :?=,+ ),+7/,(& JE6*/6*1 /6*,1 =?=,+4 71* >(1*

    ),+7/*) (1 ?&),+7/*)K

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    19/127

    !"#$% '#()*#)

    + ,- +

    .(/ 0120( 3* 0(40 0(/*/ 41/ 52*0 4 *04103)6 7#3)0 8 930(

    /)#26( :/)*#1$ ;

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    20/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'() *+,,-**.+//0 12,3* 45- *464-7-)4 4564 2* 6 /2-8 9-

    &-1-64* 45- *67- 1&',-** (245 45- *-,'): *6/-* 1-&*')8

    ;5-)< 4' 4'1 24 6// '..< (245 45- 452&: *6/-* 1-&*') 5-

    :'-* *'7-452)= */2=54/0 :2..-&-)48 >)*4-6: '. 6*32)= 527

    4' *60 45&-- *464-7-)4* '+4 /'+:< 5- 6*3* 527 4' *271/0(".-545-7 2)*2:- 52* 5-6:8 ?): '),- 6=62)< -@-) 45'+=5

    45- 452&: *6/-* 1-&*') :'-*)A4 +44-& 6 ('&:< %&'()

    *+,,-**.+//0 12,3* 45- /2-8 ;52* 2* 6 =&-64 16&40 4&2,3 B

    6): ')- 4564 0'+ 4'' ,6) :' 4'' (245 76*4-&0 '.C-)*'&0 ?,+2408

    ;52* @2:-' 2* 6 =&-64 -D671/- '. 5'( C-)*'&0 ?,+240 ,6)

    E- *+E4/0 +*-: 4' =62) 6) 2)*2=54 2)4' 45- 2)4-&)6/&-1&-*-)4642')* '. 6)'45-& 1-&*')8 F245 45- 452&:

    *6/-*76)< %&'() 2* 6E/- 4' :-4-&72)- (5-) 5- 2* 4-//2)=

    45- 4&+45 (245'+4 45- *6/-*76) -@-) '1-)2)= 52* 7'+458;52* 2* E-,6+*- '. 45- *+E4/- ,56)=-* 2) 52* E':0

    /6)=+6=- 4564 %&'() ,6) -6*2/0 :-4-,4 (245 52* (-//!:-@-/'1-: *32// '. C-)*'&0 ?,+2408 ;5&'+=5 45-

    G+-*42')* 5- 6*3*< 5- 2* 6E/- 4' ,6/2E&64-< +*2)= C-)*'&0

    ?,+240< (5-) 45-0 6&- 4-//2)= 45- 4&+458 %&'() 45-)

    *271/0 12,3* 45- H':: ')- '+4I J28-8 45- 6)*(-& (5-&-45- 1-&*')I* 150*2'/'=0 2* )'42,-6E/0 :2..-&-)4K 6): 5-,6) *6.-/0 12,3 4564 6* 45- /2-8

    >4 2* 271'&46)4 4' &-7-7E-& 4564 C-)*'&0 ?,+240 7+*4E- ,6/2E&64-: ') 6 1-&*')!E0!1-&*') E6*2*8 ;5-

    :2..-&-),-* )'42,-6E/- 2) ')- 1-&*')A* 150*2'/'=0 4564 2*

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    21/127

    !"#$% '#()*#)

    + ,- +

    ./""0)1 .(/ .23.( 40"" )#. )/5/**620"$ 7/ .(/ 5(6)1/*

    6)#.(/2 8/2*#)9* 8($*0#"#1$ 4#3"% *(#4 :#2 .(/ *6;/

    0)./2)6" 2/82/*/).6.0#) .(/ /6*0/2 0. 1/.*< ?#3 56)

    8265.05/ @/)*#2$ A530.$ #) AB?CBD

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    22/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    !"#$%&'(" *+ ,-../01

    %&''()* +, ,+-'./ & ')(01,, (2 .+3+456 7&81 /(9 181)

    ,114 ,(-1(41 ,( :11' +4 0(481),&*+(4 *;&* *;1/ &)1

    ;&45+45 (22 181)/ ) ?114

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    23/127

    !"#$% '#()*#)

    + ,- +

    .(/* 012()/341 /* #)1 #5 0(1 67#21**1* 0(80 9/"" :8;1

    0(1 8)%

    /) $#47 2#::4)/280/#) 9/0( #0(17* /) =1)178"? .(1

    *##)17 $#4 28) 8""#9 0(1 #0(17 617*#) 0# 0#08""$ 8)%

    2#:6"101"$ 71"8@ 8)% 511" 2#:5#708 (#9 %# $#4

    34/2;"$ :#A1 2"#*17 0# 0(1: "/;/)= 8)% 074*0/)= $#4B

    .(/* /* 9(171 /0

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    24/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'()*+,- .(/ +, /0,12 3*' 1*(,1'/ (&' )*45-* )*()

    )*4/' 6'467' (&',8) )&(+,'9 :&(1)+)+4,'&/ 4; 06,4)+/)/? %'1(5/' )*+/ +/ .*() @5/) *(66',/ ,()5&(7702

    3*' )*+,- +/ )*45-*? )*() +) 1(, )(A' BC#D E+,5)'/ )4

    *(66', ,()5&(7702 F,1' 045 /)(&) )4 -') -449 () )*'

    6&41'// 4; G(664&) 045 1(, %&+,- )*() )+E' 94., )4

    HCID /'14,9/ .+)* '(/' )4 /+E57)(,'45/70 1&'()' (

    ;''7+,- 4; )&5/) (,9 14E;4&) %'1(5/'J

    3&5/) K L4E;4&) M G(664&)

    G(664&) +/ E(9' 56 4; ).4 E(+, 6&41'//'/? :(1+,- (,9

    ='(9+,-2 :(1+,- +/ .*'&' 045 174/'70 &'67+1()' 6(&)/ 4;

    )*' 4)*'& 6'&/4,N/ 14EE5,+1()+4, /4 )*() )*'0 %'-+, )4

    &'14-,+/' 6(&)/ 4; )*'E/'7O'/ +, 0452 ='(9+,- &';'&/ )4

    94+,- /4E')*+,- ,'. .*', 045 (&' +, G(664&) .+)*(,4)*'& 6'&/4, )*() .+77 1(5/' )*' 4)*'& 6'&/4, )4

    &'67+1()' 0452 P5) %';4&' 045 1(, :(1' /4E'4,' (,9

    /)(&) )4 -') +, G(664&) .+)* )*'E? 045 ,''9 )4 A,4.

    .*() )4 744A ;4&2 3*'/' (&' )*' E(+, )*+,-/ )4 .()1* ;4&J

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    25/127

    !"#$% '#()*#)

    + ,- +

    !"#$%&'&(#)

    .#*/01232*/012*45675" 28912**7#)*:"7);7)

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    26/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&' ()*+, -&(. %/+)&.- 01(*2-' 23/ 4/- *) 2& 56-

    622/+2*&+ 2& 23/ &23/. 5/.)&+7 8&9 6./ 23/- 3&:;*+,

    23/*. )3&(:;/.) &. )*22*+, *+ 23/*. 136*. ?&

    23/- @&A/ 23/*. 36+;) *+ 1/.26*+ 96-) 6) 23/- 26:4

    6B&(2 1/.26*+ *;/6) 0./ 23/- B./623*+,

    ):&9:- C.&@ ;//5 *+ 23/*. )2&@613 &. D(*14:- C.&@ 23/

    2&5 &C 23/*. 13/)2 0./ 23/- 26:4*+, ./6::-

    D(*14:- E) 23/*. A&*1/ ./6::- )&C2 0./

    23/./ )5/1*C*1 9&.;) 23/- ()/ C&. )5/1*C*1 23*+,) ?*; -&( ,& 2& 23/ )6@/ )13&&: &. ;& -&( 5:6- 6

    )*@*:6. )5&.2 E

    23*+4 -&( ,/2 23/ *;/6G

    %& -&( 6./ 5.&B6B:- 23*+4*+,' H23*) )&(+;) ,./62 B(2

    3&9 ;& E ()/ I655&.2>H J+1/ -&( 36A/ ()/; -&(.

    %/+)&.- 01(*2- 2& 9&.4 &(2 93/./ 23/- 6./' 23/ )/1&+;)2/5 *) 2& B/,*+ 2& @6213 6+; @*..&. 2362 B/36A*&(.

    93*:)2 1&@@(+*162*+, 9*23 23/@7 0) -&( 1&+2*+(/

    561*+, 23/@ @&./ 6+; @&./' -&( 9*:: ;//5/+ I655&.2

    9*23 23/@ 6+; 23/ 1&++/12*&+ 23/- C//: 2& -&(7

    !"#$%&'(K6213*+, 23/*. B/36A*&(. *) 93/./ -&( ./5/62 23/*.

    B/36A*&(. B614 2& 23/@ *+ 23/ )6@/ 96- 6) 23/- 6./

    ;&*+, *27 L&. /F6@5:/' *C 23/- 36A/ 23/*. .*,32 :/,

    1.&))/; &A/. 23/*. :/C2 :/,' 23/+ *C -&( 9/./ 2& 1.&))

    -&(. .*,32 :/, &A/. -&(. :/C2 :/, 23/+ -&( 9&(:; B/

    K6213*+, 2362 56.2 &C 23/*. 53-)*&:&,-7

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    27/127

    !"#$% '#()*#)

    + ,, +

    !"##$#"%&

    -.//#/.)0 1(2./ 32(45.#6/ .* 7(2/2 $#6 /28241 1(2./32(45.#6/ 349: 1# 1(2; 4* .< $#6 72/2 1(2./ /2

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    28/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    !"#$%&'(&) %&' ()* +(,-./012 +/))&)/01 &) -)&33&4*)

    +/))&)/01 (3 5&01 (3 6.(, 7&' ()* 8&/01 3,(73 &', &9 ,.*

    &,.*) :*)3&0;3 -&03-/&'3 (6()*0*33< =9 7&' ()* 3/+:57

    -&:7/01 ,.*/) >*.(4/&') ,.* +&+*0, ,.*7 8& /, ,.*0 7&'

    ()* +/+/-?/01 ,.*/) >*.(4/&') @ A/+/-?/01 8&*3 ,.*

    )*4*)3* &9 6.(, 6* 6(0,B C.(, 3(/82 = .(4* >**0 '3/01

    D(::&), 9&) +(07 7*()3 0&6 (082 6/,. ,.* *E-*:,/&0 &9

    &0* &) ,6& FGH ,)(/0*8 9)/*0832 = .(4* 0*4*) #5#'>**0

    -('1., &', '3/01 ,.*3* 3?/553 0(,')(5575*9**5/01 ,.(, /3 .()8 ,& :', /0,& 6&)83 @ ,.&'1. 6.*0

    7&' >*1/0 ,& :)(-,/-* D(::&), 7&' 6/55 3&&0 ?0&6 6.(,

    = +*(0< L.*0 7&' 3,(), &', '3/01 ,.*3* ,*-.0/I'*32 ,./3

    9**5/01 6/55 >* .()8*) ,& 8*,*-,2 >', &4*) ,/+*2 7&' 6/55

    >* (>5* ,& )*-&10/3* /, +&)* I'/-?57 (08 *(3/57*.(4/&') C$ 865#713/.* DB$ 861E 0.1. 03/(

    *(#: 6**3*/6)/* 6)% /(. 631"3). */688 /(6/ =(.=2 3) $#71B64*> ? 8#7)% /(6/ F7*/ B$ 4.//3)4 3)/# 96::#1/ 6"" #8 6

    *7%%.) /(.$ 0#7"% B. /1.6/3)4 ;. G73/. %388.1.)/"$> ?H%

    B. #88.1.% *.6/* 03/( ;#1. ".4 1##; D?H; G73/. /6""

    6)$06$E< /(.$H% /71) 6 B"3)% .$. 38 ;$ B64464. 06*

    *"34(/"$ #5.1 /(. "3;3/ 6)% #8/.) 3) *(#:* ? 0#7"% B.

    #88.1.% 6 %3*=#7)/> I71.< 3/ 06*)H/ .5.1$ /3;.< B7/ 03/(

    :16=/3=. 6/ 96::#1/ ?H5. 8#7)% /(. 1.061%* #)"$3)=1.6*.> A8/.1 6"" J 0(# 0#7"%)H/ 06)/ /# B. (.":3)4

    :.#:". /(6/ /(.$ "32.K

    ? 1.;.;B.1< )#/ "#)4 68/.1 ? 831*/ ".61)/ 6B#7/ 96::#1/ ?

    06* 3) 6 /.6; ;../3)4 03/( 6 :.1*#) 0(# F7*/ 0#7"%)H/

    */#: /6"23)4> L5.1$#). 06* "##23)4 61#7)% 6/ .6=(#/(.1 03*(3)4 *(. 0#7"% */#: 6)% ? /(#74(/ /# ;$*."8< ?

    0#)%.1 38 96::#1/ =#7"% (.": (.1.K I# ? :63% 6//.)/3#)

    /# (.1 :($*3#"#4$< /#)6"3/$ 6)% 0#1%* 6)% B.46) /#

    ;6/=( (.1 B.(653#71> I71. .)#74(< 68/.1 6B#7/ /0.)/$

    *.=#)%* ? B.46) /# 8.." /(6/ 86;3"361 8.."3)4 #8 96::#1/

    3) ;$ B#%$> A)% /(.) ? %.=3%.% /# (65. *#;. 87)> M#

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    30/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'(& )*&+ , -.%& (.//01 &+0 &*2 34 56 73%0 )*&+ 56 (*8+&

    +'71 '71 )'&9+01: ;.(0 073.8+< %+0 .7937%9*3.%=6

    (.//01 &+0 &*2 34 +0( 73%0 &33: >+07 , 1*1 *& ' %09371

    &*50 ?'71 %3 1*1 %+0@ '71 ' &+*(1 &*50 ?'71 %3 1*1 %+0@:

    A+07 , %&32201< %+0B1 73& 37=6 %&32201 +0( *790%%'7&

    &'=C*78< /.& %+0B1 %&'(&01 9+09C*78 +0( 73%0< 2(3/'/=6

    &+*7C*78 &+0(0 5.%& +'D0 /007 %350&+*78 +'78*78 3.&

    34 &+0 071 34 *&E >+*% *% -*(' &('1*&*37'= .%0 34 F'223(&

    /.& *& -.%& %+3)% )+'& 63. 9'7 13 )*&+ *& )+07 63. 80&9(0'&*D0:

    G3)< '% )*&+ 53%& 2(390%%0% *7 G0.(3!H*78.*%&*9

    I(38('55*78< &+0 &(*9C &3 /0935*78 5'%&0(4.= )*&+

    F'223(& *% &3 2('9&*90< 2('9&*90 '71 2('9&*90E >+0 53(0

    &+'& 63. 2('9&*90 &+0 0'%*0( *& )*== /09350 '71

    0D07&.'==6 ?73(5'==6 *7 37=6 ' 40) )00C% 34 937%9*3.%0443(&@ 63. )*== %&'(& &3 13 *& .7937%9*3.%=6: F0505/0(? :033+&, (+5

    )* (0* -00 -)503(+&,= (0.* -)503(+&,= 600: -)503(+&,=

    -50:: -)503(+&, )* 3.-30 -)503(+&,;

    B.@0 ?)1 0@0* >00& :+-30&+&, 3) 3(0 *.C+) .&C . -)&,

    (.- ')50 )& 3(.3 A1-3 5.D0- ?)1 600: (.77?E

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    33/127

    !"#$% '#()*#)

    + ,- +

    .(/. .(0 1#"2.232/) 3#4"% 50 */$2)6 *#70.(2)6 $#4 3#4"%

    /6800 92.( 54. .(0 )06/.2:0 ;00"2)6 $#4 (/:0 ;8#7 ?)3(#8* /80 1#908;4" .(2)6*@

    A# 9(/.B C(/. /80 /)3(#8* 4*0% ;#8 0D/3."$B

    ?)3(#8* /80 4*0;4" 503/4*0 )#. #)"$ 3/) 90 (/:0 #48

    *./.0 3(/)60% 5$ 0D2*.2)6 /)3(#8*E 90 3/) /"*# 380/.0

    )09 #)0*> F7/62)0 2; $#4 (/% /) /)3(#8 .(/. 9#4"% 14.

    $#4 2).# / 80/""$ 6##% 7##%

    !2G0E 80/""$ 38/1> H4. 503/4*0 $#4 G)09 /5#4. .(2*

    308./2) *#)6E /)% .(0 9/$ 2. 7/G0* $#4 ;00" 9(0) $#4

    "2*.0) .# 2.E $#4 3/) ?)$.(2)6 .(/. 90 3/)

    ;00" 2) #48 )08:#4* *$*.07 90 3/) /)3(#8 .# *#70.(2)6

    90 *00E (0/8E ;00"E *70"" #8 ./*.0> H4. .(080 /80 / ;09

    .823G* .# *4330**;4" ?)3(#82)6 .(/. 7/G0 2. 7#80 "2G0"$

    .# 9#8G>?. /)$ .270 .(/. / 108*#) 2* 2) /) /**#32/.0%E 2).0)*0

    *./.0 9(080 / *1032;23 *.274"4* 2* /11"20% /. .(0 10/G #;

    .(0 0D10820)30E .(0 .9# 92"" 50 "2)G0% )048#"#623/""$>

    J(0 *4330** #; /) /)3(#8 %010)%* #) .(0*0 ;2:0 .(2)6*K

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    34/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&' !"#$"%!#& () *&' (*&'+ ,'+-(.- /.*'+.01

    '2,'+/'.3' 45) *&'6 0+' '2,'+/'.3/.7 0 +'0116

    -*+(.7 /.*'+.01 -*0*' *&'. *&' 0.3&(+ /- 7(/.7 *(

    8' 9(+' '))'3*/:'; 90.07' *( +',1/30*' *&' -*/9>1>- E /) 6(>

    0.3&(+ 0 *(>3& (. *&'/+ D.'' 6(>I11 )/.B /* 9(+''))'3*/:' /) 6(> *(>3& *&'9 /. '203*16 *&' -09'

    ,103' (. *&'/+ D.''G 0- (,,(-'B *( *(>3&/.7 *&'9

    0 )'? /.3&'- *( *&' 1')* (+ *&' +/7&*;< 0.B

    %&' ")'2$, () */9'- 6(> 0.3&(+ /* 4%&' 9(+'

    */9'- 6(> 0.3&(+ /*G *&' 9(+' '))'3*/:' *&'

    0.3&(+ ?/11 8';J

    K( ?&'. 6(> ?0.* *( 3+'0*' 0. 0.3&(+ /. 6(>+-'1) 4A+ /.

    -(9'(.' '1-'; *&'-' 0+' *&' D'6 )03*(+- *( 8' 0?0+' ()J

    L- 9'.*/(.'B 08(:'G (.' () *&' D'6- *( ->33'--)>1

    L.3&(+/.7 /- */9/.7J %&/- /907' -&(?- 0 7+0,& ?/*& *&'

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    35/127

    !"#$% '#()*#)

    + ,- +

    ./*0 1)% 21"" #2 1) 03#4/#)1" *4140 50/)6 .07.0*0)40% 5$

    4(0 5"189 8:.;0< =#: >1)4 4# 506/) ?)8(#./)6 1* *##) 1*

    $#: *00 4(03 6#/)6 /)4# 4(0 *4140 1)% *4#7 @:*4 1240.

    4(0$ 7019 /) 4(14 *4140< A($B C081:*0 >0 1.0 41"9/)6

    15#:4 *31"" 8(1)60* /) 4(0 #4(0. 70.*#) $#: >/"" )00%

    .01""$ 6##% D0)*#.$ ?8:/4$ 4# *7#4 4(0 8(1)60*< E;0)

    >/4( 131F/)6"$ 6##% D0)*#.$ ?8:/4$ 4(0.0 >/"" )01."$

    1">1$* 50 1 *"/6(4 %0"1$ 504>00) 4(03 0)40./)6 4(0

    *4140 1)% $#: )#4/8/)6 4(03 0)40./)6 4(0 *4140/*0G >(0) 4(0 *4140 *4#7* /)40)*/2$/)6 $#: >/""

    9)#> 5081:*0 $#:. D0)*#.$ ?8:/4$ *7#4* 4(0 8(1)60* /)

    4(0/. 7($*/#"#6$< ?)% 5$ 4(0 4/30 $#: )#4/80 4(14 /4 /*

    %.#77/)6 1)% .0"01*0 4(0 1)8(#. /4 >/"" (1;0 1".01%$

    70190% 1)% 50 #) 4(0 %08.01*0< H(/* %/16.13 50"#> >/""

    3190 /4 3#.0 8"01. (#> 4# 4/30 4(0 177"/814/#) #2 1)1)8(#.I

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    36/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'() *' +,-. /('- 0.*123 +(45) 6(74 *8,&,-6 -( 23- ,1-(

    9*::(4- +,-. -.3 (-.34 :34'(1) *1; -( 3 E*,1 9*::(4- +,-. -.3 (-.34 :34'(1C#> %'5 ,= ,- ,' (5*6 =(4 6(7 -( -(70. -.3/ (1 -.3

    51705&3 ;74,12 -.,' 3 I33: ,1 /,1; -.3 =*0-(4'

    /31-,(13; *8(B3 +.31 0.((',12 6(74 '-,/7&7'

    -( /*

    M*B3 -.3 (-.34 :34'(1 43/3/834 * 43*&&6,1-31'3 :*'- 3

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    37/127

    !"#$% '#()*#)

    + ,- +

    *./0 (1.2 /(.3 $#4 (1.2% .)% 21.""$ 511" 3(1

    511"6)7* #5 816)7 3#3.""$ 9#)56%1)3:;? @* 3(1$ 21A1A812 3(1 B.*3 1CB1261)91 4*1$#42 D1)*#2$ @9463$ 3# )#3691 .* 3(1$ 8176) 3#1)312 3(1 %1*621% *3.31 #5 816)7 E#)56%1)3?

    F(1$ A.$ 1)312 63 G469H"$0 #2 *"#/"$0 63 %#1*)I3A.3312 .* "#)7 .* $#4 .%J4*3 $#42 36A6)7

    .2#4)% 3(1A? @* 3(1$ 1)312 3(1 *3.310 *3.23.BB"$6)7 . *36A4"4* KD# *3.23 3# 562A"$ B21** #)#)1 #5 3(162 H)49H"1*0 *.$ #) 3(162 3(4A8? F(6*

    B21**421 %#1*)L3 )11% 3# 81 M12$ (.2%0 J4*3

    1)#47( B21**421 *# 3(1$ 9.) 511" 63

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    38/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! ## !

    $% &'!())*+ ,-' (./-01 (.2 34' +031 5'.401+6/37,+ ,0 ,'4, 8-',-'1 ,-'+ 1',31. ,0 ,-' 4,(,'

    9:; ,-'+ 1',31. ,0 ,-' 4,(,' ,-'.

    ?&6@AB6@:=>5C +03 -(D' 43//'44;3**+

    7.4,(**'2 (. (./-01C :; +03 272.E, .0,7/' (

    /-(.F' ,-'. /-'/G ,-(, ,-'+ (1' 7.,'.4'*+

    (440/7(,7.F 87,- ,-' 4,(,' (.2 H(G' 431' +031

    4,7H3*34 74 3.7I3' '.03F-% :; 7, 74 7.,'.4' (.2

    3.7I3' '.03F- ,-'. 1')'(, ,-' )10/'44 4'D'1(*,7H'4 (.2 ,-' 1')',7,70. 87** (4474, 7. J37*27.F

    ( 4,10.F'1 (./-01K%

    50 ,-(,E4 7,L +03 .08 -(D' ( 103F- 72'( (J03, -08 ,0 34'

    ,-' >BM )10/'44 0; 6./-017.F% @-74 74 1'(**+ N34, ,-'

    J'F7..7.F 0; 6./-017.F J3, 7, H(G'4 ( 1'(**+ F002

    4,(1,7.F )07.,% >08 7, 74 ,7H' ,0 ;7.2 +0314'*; ( 87**7.FD0*3.,''1 (.2 (4G 7; ,-'+ 87** (**08 +03 ,0 )1(/,7/'

    6./-017.F 87,- ,-'H% @1+ 40H' )047,7D' 4,(,'4 *7G'

    6./-017.F ( ;''*7.F 0; /0.;72'./'L ( ;''*7.F 0;

    'O/7,'H'., 01 'D'. ( ;''*7.F 0; J*744% :; +03 /(. ;7.2 ,-'

    80124 ;01 7, ,-'. +03 /(. ;''* 7,C

    64 +03 J'/0H' H01' )1(/,7/'2 (, 6./-017.F +03P** ;7.2

    0,-'1 ,'/-.7I3'4 *7G' 4,(/G7.F (./-014L /0**()47.F

    (./-014 (.2 /-(7.7.F (./-014 (1' (** )08'1;3* ,00*4 7.

    +031 ,00*G7,%

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    39/127

    !"#$% '#()*#)

    + ,- +

    "#$%&'$ (')*+,-

    "$.-/ 0,%. 1$+,2'/

    !" $%&'()* +(,*-.) (,* /0', 1)%,*.)

    .(/* 0##1 /* 2) 3453""3)6 672)*57/86 97#: #)3 #9

    6(3 9/7*6 "356;73* #) .(373"3?2)6 /)9#7:26/#) 20#;6 @)5(#7/)A *6276*

    6#B27%* 6(3 0#66#: #9 =2A3 C->

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    40/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    !"#$%&'(" *+ ,&-./% ,/0"-

    %&' ()*+,- (,.'* /01 23'0+'. 45 60-.*'3 7 83)-.'39

    :,**,/)-; +&')3 1+)-;9 +&0+ 0**,/1 5,< +, @,3' '01)*5 )-.)-; +,? %&)1 )1 ).'0* :,3

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    41/127

    !"#$% '#()*#)

    + ,- +

    .)% $#/0"" 12#343"$ )#5678 (#9 65 6* 926558) :/*5 "6;8 4

    "#5 #< 5(6)=* $#/ >4$ (4?8 284% 38>/)6745#2* "6;8 A4256) !/5(82 B6)= '2CD

    E6"" F"6)5#) 4)% E4247; G34>4C

    H(8 A6"5#) A#%8" 1424=241( >4;8* /*8 #< >6)% 284%*

    IJ828 $#/ 284""$ 9#)%826)= 9(45 5(8 A6"5#) A#%8" 6*

    4)% 9($ $#/028 =#6)= 5# *5425 /*6)= 65 *##)KLD "#*5182456?8* IH(8 A6"5#) A#%8" 6* 6>1#254)5D

    477#2%6)= 5# 9(#>KLD 74/*8 M 81#254)5 65

    6*KLD 128*/11#*656#) I65 6* 4**/>8% 5(45 $#/0"" *5425 /*6)=

    65 *##)NLD 54= O/8*56#)* I(4?8)05 5(8$K 7#/"%)05 $#/KLD

    4)% 4 7#/1"8 #< #5(82 145582)* 5##D :/*5 #28 *8)*8 4* $#/ 8P1"#28 5(8 A6"5#) A#%8"

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    42/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&' ()*+,- (,.'* )/ 01.' 23 ,4 5,26&*7 +8'-+7

    31++'5-/9 :1;& 31++'5- &1/ '-.*'// 3,//)

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    43/127

    !"#$% '#()*#)

    + ,- +

    !"#$ &'()"(*+$,-'

    . "#*/ 0123#245/671 6* 8(1) 81 02#76%1 5 75"91

    :9%;141)/ 86/(#9/ /1""6); /(1 "6*/1)12 8(# 45%1 /(5/

    75"91 :9%;141)/< =/ 8#2>* *# 81"" ?1@59*1 21"5/671

    8#2%* 521 215""$ 15*$ /# 5;211 86/(< A# (#8 %#1* 6/

    "##>B

    C5*6@5""$ 5)$ 8#2%* /(5/ 521 75"91 :9%;141)/*

    *9@( 5*D ;##%E ?5%E ?1//12E 8#2*1E *"#8E 35*/ 1/@6);E #2 *"#8 @#40521% /# 26%6); 5

    (#2*1 #2 *"#8 @#40521% /# 26%6); 5 ?6>1 ?9/ 6/ @#9"% ?1

    *"#8 @#40521% /# 5 G122526 #2 5 :1/ 0"5)1 #2 5

    4#/#2?6>1< C1@59*1 /(1 *1)/1)@1 45>1* *1)*1 ?$ 6/*1"3

    /(1 "6*/1)12 6* "6>1"$ /# 5@@10/ /(1 /(#9;(/E *9@( 5* /(1

    @52 6* K96/1 *"#8E 5)% 63 /(1$ 5@@10/ /(5/ /(#9;(/ /(1)

    /(1$N"" ?1 4#21 "6>1"$ /# ?5*1 /(162 /(6)>6); #) 6/ 392/(12

    6)/# /(1 @#)712*5/6#)

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    44/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    !"#$% ' ())%*+

    %&'() * +,,)-. /( .0) 1&2) 3) '() 30)1 3) /2456 .0&.

    71) .0/18 -&'()( &17.0)9: ;0) 0'2&1 2/1< =>?+@

    9)&(71( &1< .0/( /( & 89)&. 3&6 .7 497A/

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    45/127

    !"#$% '#()*#)

    + ,- +

    !"#$%&' )*+,-.%&/0&

    .#/0"12 345678"1)91 6* 8 :8$ #; *5

    $#5 9#/6)< =# 8"" =(1 =@86)6)< *1**6#)* /18)* =(8=

    $#5?@1

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    46/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'()(*(+,- .+,/ %+01,)-2 )3(,4 +5&6) )31 ,(73) *'65

    18+9%'1: )& &;10!71,10+'(-1 + -()6+)(&, 1> ?@&)(,7

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    47/127

    !"#$% '#()*#)

    + ,- +

    ./0*122#*343#)* 5/0 (160"$ 1*071" 8(0) 45"93)6 4#

    20#2"0: 5* 4(0$ 5""#8 $#1 4# 3)7"10);0 4(0 @7 $#1 80/0 *0""3)6 5 ;5/ $#1 ;#1"%

    2/0*122#*0 4(54 4(0 A1$0/ ;5/0* 5A#14 4(0 *5704$ #7

    4(03/ 25**0)60/* B*# ?(0 90$ 3* 4(54 4(0 8#/%

    ?(0 *0;#)% 25/4: "5;93)6 5 /070/0)435" 3)%0D:

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    48/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&' ( )*(*+,+-* *& .+ ( /-01+')(2 34(-*050+' 0* ,4)*

    6&-*(0- 789: ( /-01+')(2 ;+-+'(20)(*0&- 2(6? (

    '+5+'+-*0(2 0-@+AB C* 6(- 2&&? )&,+*D0-E 20?+ *D0)F

    G1+'H*D0-E 5++2) 20?+ *D(*B

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    49/127

    !"#$% '#()*#)

    + ,, +

    -./(0122%34)0

    5462 7#8 294:;"2 0(2*2 0#? *(#?"% /# 0# 0(2 @#*0 A77.B2C>#? B#?"% /# 0# 0(2 @#*0 A77.B2C

    5(2$ 722" D28$ %.77282)0C 5(2 7.8*0 *0402:2)0 ?*2* 4-#%4" A;2840#8 #7 12B2**.0$ E*(#?"%F

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    50/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'&%&()& *+& ,%-)&.. /( 0 102 *+0* 0..34&. /* +0. 5&&(

    )-4,6&*&78

    9( .06&. 2-3 402 +0:& 0 )6/&(* *+0* *&66. 2-3 *+&2 0%&

    ;7&)/7/(

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    51/127

    !"#$% '#()*#)

    + ,- +

    !"# %&'()*+,

    .(/* /* #)0 #1 2(0 03*/0*2 3)% 4#*2 15) 6/"2#) 6#%0"

    732208)* 2(32 $#5 93) *2382 5*/): *283/:(2 3;3$< = 23:

    >50*2/#) /* "/2083""$ 3%%/): 3 8(02#8/93" >50*2/#) #)2#

    2(0 0)% #1 ;(32 $#5 *3$? /*)@2 /2A B(0) $#5 *2382 5*/):

    2(04? 70#7"0 C5*2 1/)% 2(04*0"D0* 3:800/): ;/2( $#5?

    %#)@2 2(0$A =)% 2(0$ %#)@2 0D0) E)#; ;($? %# 2(0$A

    .(0 15))$ 2(/): /*? 4#*2 70#7"0 3"803%$ 5*0 2(0*0F

    8/:(2A

    G080 380 3 H5)9( #1 0I347"0*J

    F(3D0)@2 $#5AF%#)@2 $#5AF;#)@2 $#5AF93)@2 $#5AF8/:(2A

    -"./ +0 1'0'2',)*"3 4,5'6

    .(0*0 732208)* 380 803""$ 5*015" H0935*0 2(0$ 380 C5*2 *#

    D3:50K =)$ 7(83*0 2(32 %#0*)@2 80108 2# *#402(/): /)

    7382/95"38 "39E* 3 801080)2/3" /)%0I< .(/* /* ;(0) $#5

    80108 2# *#402(/): /) 3 ;3$? 2(32 43E0* /2 D08$

    34H/:5#5* 3* 2# ;(32 $#5 380 39253""$ 801088/): 2#< L#8

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    52/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'()*% +,-./ 01% 2345 6-078 601%(78 601'08 601-./78 7,0+997

    %0:;

    %(%(?%4 01'0 01-./ 01'0 2% *%90 ?%1-.5 '0 01'0

    )*':% .%'4 01'0 )%4,3.@ A

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    53/127

    !"#$% '#()*#)

    + ,- +

    "#$%&'&()*+ ,+-+()#.

    . /#012324567 87"745#) 5* 673$ *505"23 4# 2 !#*4

    973:#3024567 ;(737 2 4 54 5* )#4

    *174 ;(24 54

    5* =75)? 4 54 %5%)G4 ?# 2* ;7""@

    A(7) $#> 02B7 2 *7 ;(7) 43$5)? 4# =7

    173*>2*567@

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    54/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    !"#$ &'(($)* +,-$(.$)#$

    %&' ()*' +,--'./ 012'-3'.*' 2)//'-. 34 5.' 56 /&'

    7,3*8'4/ 2)//'-.4 65- 9'//3.9 )9-'':'./ 5- 2,//3.945:'5.' 3./5 ) /-).*' *5.;'-4)/35.)

    G'-' )-' 45:' '1):2

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    55/127

    !"#$% '#()*#)

    + ,- +

    *#./0(1)2 0(30 1* 4#**15"/6 570 )#0 %/81)10/"$ 097/6 3)%

    (3:/ 0(/ #0(/9 4/9*#) 9/3%1"$ 329//;

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    56/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'() *&' +,-.-, /& /01- /2- 345 &'/ /&5462/ &,

    /&7&,,&8 7&,54569:& *&' 805/ /& 6& /& 3-) 5&8 &, 45 # 745'/-;9%&'() *&' +,-.-, /& 3'* /24; 45 3('- &, ,-)92&4>-@ ,-60,)(-;; &. /2-4, >2&4>-@

    /2-* 0,- 47+(4>4/(* 06,--456 /& ;&7-/2456 -(;-A

    B-60,)(-;; &. 82-/2-, /2-* /01- /2- 345 &'/ /&5462/@ &,

    /&7&,,&8 7&,5456@ /2-* 0,- ;/4(( 06,--456 /& /01- /2-

    345 &'/ 82-5 /2-* 05;8-,A B-60,)(-;; &. 82-/2-, /2-*

    +,-.-, /& 3'* 4/ 45 3('-@ &, ,-)@ /2-* 0,- ;/4(( 06,--456 /&

    3'* 4/A

    !"#$%&'()*"#(+ -"').+()%

    C >&5?-,;0/4&50( +&;/'(0/- 4; 82-,- *&' /01- 0

    >&7705) D4A-A ;&7-/2456 *&' 805/ /2- (4;/-5-, /& )&E

    05) /',5 4/ 45/& 0 F'-;/4&5A =24; 6,-0/(* ;&./-5; /2->&7705) 05) 84(( 45>,-0;- >&7+(405>-A G;;-5/40((*@ 4/

    0?&4); /2- .--(456 &. 0'/2&,4/0,4054;7 /20/ >05 >&7-

    .,&7 /-((456 ;&7-&5- 820/ /& )&A

    GH07+(-; &. >&5?-,;0/4&50( +&;/'(0/-; 0,-I

    J05 *&' >(&;- /2- )&&,9

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    57/127

    !"#$% '#()*#)

    + ,- +

    .#/"% $#/ 01)% *12)1)2 3(1* %#4/05)36

    .(73 785 3(5 4(7)45* #9 $#/ :14;1)2 /: 0$4"#3(5* 98#0 3(5 %8$ 4"57)58 #) $#/8

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    58/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! #$ !

    %&'()*)+, -)./0') &( &'*1( 230 '/2&*4 &( (3 (5)6, &( &' (5)

    7)+'3* 853 &' -)&*4 903(): '/2&*4 &( (3 / $+: 7/+(2;

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    59/127

    !"#$% '#()*#)

    + ,- +

    ./%0 (1* %021*1#)3 456 "##71)8 9/27 (0 :1*(0%

    (0 (/% ;5*6 8#)0 0) 6(#58( (0 0* (1. *# .52( 61.0 /)% ./70* 6(1)8* *#

    .52( 0/*10=

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    60/127

    !"# %#&'#() (* +#,'-.-/ 0#1'*2+.-/1.)(.& 3'*/',44.-/

    ! ## !

    $ %&' %&( )&*+ ,++-.(/012+3+34+'5 6)+ 7&--0 )&*+ +&'0 .( 6).0 8,,.%+9:87 ;8 6)+ %)&.' ,++-0 =(;+' + 0+(0+5 7)+(

  • 8/12/2019 The Secrets to Learning NLP by Lloyd Johnson Copy

    61/127

    !"#$% '#()*#)

    + ,- +

    *.$/)01 23 4#567* 3(78 3# 35$ .)% 8.97 *7)*7 #4 /3 .3 .)

    :)6#)*6/#:* "7;7"< .)% .* 3(7$ %# *#< /3 ).3:5.""$

    %77=7)* 3(7/5 "7;7" #4 35.)671 >(757 .57 4#:5 3$=7* #4

    .8?/0:/3$ *=76/4/6.""$ 3(.3 %77=7) 35.)67 7;7) 4:53(75@

    !"#$#%#&'()% +,-'&.'/0

    >(/* /* A(7) $#: :*7 3A# A#5%* 3(.3 *#:)% 3(7 *.87