From 192cec84689200ae81df4c31708853dd13ddcdad Mon Sep 17 00:00:00 2001 From: Anthony Marakis Date: Sun, 16 Jul 2017 18:28:17 +0300 Subject: [PATCH 1/5] Update nlp.py --- nlp.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nlp.py b/nlp.py index 37f62c779..9e3e87fec 100644 --- a/nlp.py +++ b/nlp.py @@ -1,4 +1,4 @@ -"""A chart parser and some grammars. (Chapter 22)""" +"""Natural Language Processing; Chart Parsing and PageRanking (Chapter 22-23)""" # (Written for the second edition of AIMA; expect some discrepanciecs # from the third edition until this gets reviewed.) @@ -23,8 +23,8 @@ def Rules(**rules): def Lexicon(**rules): """Create a dictionary mapping symbols to alternative words. - >>> Lexicon(Art = "the | a | an") - {'Art': ['the', 'a', 'an']} + >>> Lexicon(Article = "the | a | an") + {'Article': ['the', 'a', 'an']} """ for (lhs, rhs) in rules.items(): rules[lhs] = [word.strip() for word in rhs.split('|')] @@ -96,8 +96,8 @@ def __repr__(self): N='man')) -def generate_random(grammar=E_, s='S'): - """Replace each token in s by a random entry in grammar (recursively). +def generate_random(grammar=E_, S='S'): + """Replace each token in S by a random entry in grammar (recursively). This is useful for testing a grammar, e.g. generate_random(E_)""" import random @@ -111,7 +111,7 @@ def rewrite(tokens, into): into.append(token) return into - return ' '.join(rewrite(s.split(), [])) + return ' '.join(rewrite(S.split(), [])) # ______________________________________________________________________________ # Chart Parsing From bcb95f2bbbdab74d25c78845dede39b1278d1994 Mon Sep 17 00:00:00 2001 From: Anthony Marakis Date: Sun, 16 Jul 2017 18:29:06 +0300 Subject: [PATCH 2/5] Update test_nlp.py --- tests/test_nlp.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/test_nlp.py b/tests/test_nlp.py index 8572eabff..6623162bc 100644 --- a/tests/test_nlp.py +++ b/tests/test_nlp.py @@ -4,20 +4,31 @@ from nlp import loadPageHTML, stripRawHTML, findOutlinks, onlyWikipediaURLS from nlp import expand_pages, relevant_pages, normalize, ConvergenceDetector, getInlinks from nlp import getOutlinks, Page, determineInlinks, HITS -from nlp import Rules, Lexicon +from nlp import Rules, Lexicon, Grammar # Clumsy imports because we want to access certain nlp.py globals explicitly, because -# they are accessed by function's within nlp.py +# they are accessed by functions within nlp.py from unittest.mock import patch from io import BytesIO def test_rules(): - assert Rules(A="B C | D E") == {'A': [['B', 'C'], ['D', 'E']]} + check = {'A': [['B', 'C'], ['D', 'E']], 'B': [['E'], ['a'], ['b', 'c']]} + assert Rules(A="B C | D E", B="E | a | b c") == check def test_lexicon(): - assert Lexicon(Art="the | a | an") == {'Art': ['the', 'a', 'an']} + check = {'Article': ['the', 'a', 'an'], 'Pronoun': ['i', 'you', 'he']} + assert Lexicon(Article="the | a | an", Pronoun="i | you | he") == check + + +def test_grammar(): + rules = Rules(A="B C | D E", B="E | a | b c") + lexicon = Lexicon(Article="the | a | an", Pronoun="i | you | he") + grammar = Grammar("Simplegram", rules, lexicon) + + assert grammar.rewrites_for('A') == [['B', 'C'], ['D', 'E']] + assert grammar.isa('the', 'Article') # ______________________________________________________________________________ From 0ef1d1f4fa7c8bd4abec1db808a08566d17ea038 Mon Sep 17 00:00:00 2001 From: Anthony Marakis Date: Sun, 16 Jul 2017 18:29:54 +0300 Subject: [PATCH 3/5] Add files via upload --- images/parse_tree.png | Bin 0 -> 13655 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 images/parse_tree.png diff --git a/images/parse_tree.png b/images/parse_tree.png new file mode 100644 index 0000000000000000000000000000000000000000..f6ca87b2fb76fe83e5b90453e8513eba14ecf24b GIT binary patch literal 13655 zcmc(Gby!s0_VCcsEe!%n!+_Ef0wU5S4m}_s-5^K|-4aT7Nl6R}NQpzYG>mk24jnV_ zjqiJZ_ugN9e}3Qdob$|{y<)G}XP>>++9yg&Ly4Gxo&W#<5G%iYt^)vIg3zyfHgy;4N?V_;xlVq#)pVPRuq!-@ZqCJk2p9uI5|1FxVX5vxp{ba zczJpG`1ttw`2_?7fIy(2prDYD(BsFCpFDXYEG#S{A|fg(Dkdf-E-o%1At5O#`Sj^i zDJdywX=xc58ChA`XV0F=$;rvf%PS}-Jb(UNQBm>5ix)~tN-tl&R9042QBhG)qVBqm7bp7>({SAAdtSkzJY;(p`oFXk&&^nv5AR^ zsi`R#3^p?}GdDMX^X83(g@vW1rInS{+qZA6t*vcrY;0|9?dlq=?fw4!dmkSkUteE8KRJ#$U0vPx@89d|>l+#x8XFs%nwpxMn}7WH(bCfL^XE?(4A$D(+Sby zsi`SA9RBTC-QB&tz5V_DgM)*^!^5MaBLo6*e0+Rza&mfldUkepetv#&adCNh zd3AMleSLj%bAv=8Q7F{y?d{#&-Ac8IKl&NKcX?^x4gip}|G6=!xl{ttopc_G`X0K@ zZ#}%s-K+pE=C)2A0!~&Q4}=8-1%yPO+d|I(0HI#x=W?&SP4}~jv-HNVdsyi4*~_$e z>M3GFdk40{*DCMj&wa`yi%zHyHBKs@s|qjGib^8uGnyw@9|FHGe;GO!!?h;8{rKDM zaLtsv>D!Ggf75e=%?kga6O`ZJZKbEi>fpj+&4r;aw1tLCt~%oK(eLL-3I{d_z9=~i zJ2Mg-tlkN1<5)>GOZ;(azg{S2>~d8NZ+y6xAa!bQKqH7;&oyRW`+z*zeg8d%w$D?( zs`n{O^dQ*OL~J!=+Cmk>X<3C8w3ZF7&oezy%bG{9Qif)I^nx(qx?m`=UZ4TZm~(yF zz0&VD;Tix1ae_w9L!=}iZxo&tmX+uQng6hVGv;Yb=BB=IRivU-Eop^rtKD9mjgfHC z3IlIK2IK|$WMLC{`!5A;#G?n~N%|CENlnf`$KT|tjeKh#ew?a0wv^!De@7V$5H_;O zqU8JEDYU74CQq?Ao&ugJ@=~d3)NQ08&r~a)!{S!>)*_|ORXwq6+O>Phul7KifyInr zj30*~OaK>gC03=iOmO_%fLxh%HA*C_CV~C<7ZhW}Z4W4^K22=~clP-3=KKYT{OevZ z81AY@`P*jG{bAlNfJKAo;I*V@x1P)WZ_ip$*Fh085mycT@2yHD;&|CB4x=5vDH@aQ zQ98i-Gw(x}Xg5YFKbV|V-G#8&jZg{T;!@%Tu}62BL0QtES_=tXQOACjSuu6mO>oc z!+jP{iLcDf$c1ydaoU(M&g@MsN@cU%efaoswrW2S2J>TXgSyTDoK>6hGasO&b>R-{ z!DbN;uFvEcxcWO2T}97uY=kligQ+pM;HbL1hYIYwG<6tTJHAR>ot!QZ#~n<;2Np_o z(cdJsq>QU*4cKiUE&41wuwDsvhdm4}ogp07aqIzTB^IE`(g(Zq2SwnCm3|l#<~n?A zN^2J~MECiPK7WEa^}BnY!0h&N3{O)%r+TXiRIz)YXPlTsweOl#!u5W$3PPkoEc76z zy<}xe{V;0G75K(afQguE8^z;_cFeY;-R%2X#{PnK-5<@V={WB}^oW4L9W`#;hT(da z8Gy^WVk|1+p{Z#sy4N2K<}U0FnQWwT-9>4^N@zF<95v%7^DKTZM+_|pU8gJBHS&nn zteT#JsmB#;3gY{qt^_u&Sl5sQPfLBpVwY*CNRjQrgCvZpf$HcFOV!}S=zLoeD1Lu+ zwAxYySk|NCSLV2Twh>y<0OK=zh(+n@A#OfVb#V+)`B79RM9&zE&1AgQ+E-1GE4~en z7S&36ap>rlx^XsZ+L)#~*Z^Z`x*>-qILLAsD3nn-c=EI^EPT(F?4tl}+6I zeX|ODXUreg6mB?x+Na+HZs(%xk#Ze8hG^fcxpqQ}9%=0z4&C`^Lll`hI9Rl>@|+G; z%~-0AVwhi>UuiT-wd2EhUfh?Hi1r}2+b*c9nvLt%h?R7PMy;N$^lwcssyMvM=Vg=C zzNv&Htu)MI&V78s6>qB1GmWDJQfW)7sPK4-(zjS78ao@lBQIO3Zq41~5U(dP3<1Nf z=%p*zHbWZbIZ`3grbOybe5#H+!c<|ar9Z`D90zFpoIVkwea7atNkhypEqmlINdMY3 zlc+pWbqLAhOyGHJ-kXCS=%Es%E0bP0a-=%i2`#2*BZ7CoOfLI*xLzkGC*&%feggf_ zDoVL6r4E1qjK-(q?BF3X zKnxt9k=W49>7#-3i99DAfes$B8;F&Kofc;q7jKF=^%Saq>~9>0o8%U>l$NG3m@Kc& z;m`^V;+=4M2l(ZzD!bHy$%{ei=Rl% zMBt$x_6X)CAyS%2wN*{<6teD(9*ld?oLUswkNQIz*}>z0UYNu$bVHhymx;@(-TeGC z4pQP3+R*KZpEN7lW*t18PzPP@*T!on274It$bOk!!ctG3|Nh3NM5tCr{Vy-{vgVub`4(w+5>62>5}~_&mPWpN+&3a0iuD=0a4akVe z`XmWl(`6t$oEyVYlsRxYF^kO~KP_VfaL}ldg~ZRS1-`U=xha}>#PwTkNM3DYg(stZ zT96`7hJSXMHyoxh0>sXoK<%7k-|hol=ul1bpSfTus|-%de4V&nNNX42Jdh7+|I7U2 zq^c<(PCA;YZZan0uk|#Dwxed8yhGJ9|DVfh6`jY%1Y2-&8nDRar^J`lEm%N(Nr-ez zz?G$L+&+jmu!=G4H(#r#c*c{Uh#n5fJn&o0YGQu`Kg}+!*%(5ViPR;KyXV}#SxR5% zyDi2@4m%pn-s2ux@G>s_5o;{f0ia#>Ldk#-%J2g)9xYsu&-AW%X2&Xnq~etbAur1Y zJGUh*l@2k*{p zUoT__;vYc%(>psZ7%u{&cBK@)P|2ZqRrC&*wpXb1^FivtYcNsuIJ}`^4gOVA=4x$g zU<#;>}d zg$F;qR-T+dgsfP}Nte7vvmkjU2BivA=k}8Z`bNbTfy!{8C+F>%Gw9qtP+8v1FV_CZ zFT0xiT^7l`>q%4}XGyY&78Y10l{Z9n2IAP^??YPWRST9o+_lUc$FSR}rhMk@1U}^* zR19#ja#5ZIir|7;GcUoNPzE)MPnM6XTEZGWcUUwiQJt4>osK*RevtSR{6*v$gE1*N z%t=)F<)=y!42`aJVn9hcJvK^m|6nxn?T`CNf12eEXxWDEeT03avi!HDna^kOvYhDz z*r>;;3&&V2kTQ!-V8CGKq!@N-Ku{Mto;7QMJ=j}q^EhQgy`88O2zlliZM;tN*` zBp+pp;}RlxRf$3it{BJ7Ebwa=2PL#3!PDjssTAiQCP~`UW9tecczKf)7YnlQ^J-NC zWQf#|{kY>#jvfN3DvXY>DPiAQPYcv4N))>=sML_(G~c_vWsVJpm5}zdK7bhDT|C_o z3j4_orEajki~@h}vuwsEt;L{3V-jWS_ZpYu;}A|!BD#Rhme-wIwSm&QhRARI8x*}q zVcFlz@HrsUmhQe>eWzizR1Wk#05r$XE6*9&-NLVaOeE=b0)1b3gj|c_`VkW)n60Er zUtiOGT4dx`N2?^_U61T`Fc1%f^R9Uq{0Ef^y9@^ZD_N48V=dQ-Kp3wU=ai> ztpH``H4%kU3PZ*8b>AOad}a9yn&$6PcjZ=bO-gf|7|QNu*I`3>AHXLTm2qM1_C*e8 zF$=B%c`vM`ji@U+5xyPxgD+AiR9S0a+^-60ef{x*a<86H{pgaT)fc@x#PnFLCH&By zd$Itof4f}SjPb+VZZ$C2)Ac%s)CkhG7ov&P0FiE0H6B#@v^6;ybIj`(zC6bTV_y9{ zu?xtS=r#{q6{9vd=6%aztbTK&az_13X3&T^BAR5x?9rz89;t$Q8dmk>Lmx>+JW2@k z6tzpBF5ZQ8^F@I0ZQmc+h_Ce;l~XhydQKNp`DAL{m*fnZBf3dDVs6dkAZNm$_;nH& zOuV%9AO#scjR#+=1N(oXgO8BXr;RW*Wd=%NRKJQ4s!lbf_i2^OT6r?Oo*B(|MO{N$ z)n^UEm9Kml{>$v&cYZ-FDmC4b?Vdhm(+y&$D_d=+Yz;Yu?)rbppV$MLPg!diqp*p- z`9e(mi}fpuaDhY+5X<{M&F{l*nf0ax)vcm>NrhM|DvH41MxWt`M}OQXa6IkeGDdWf4E@G`7 z@uX5Yd&u51pJigo-bUj4I>t=K(Ai5=+`LcX`@TS@_|>iZ|4_Xg?EQg$NC;oXxZ(6^ z+>Wp5mq};f^`}Ce%oP6XaEr%D68{wF{SN4T)l*k^Gk4lQB-{qQ7dl-}d?+W_pD^1vki(^;zAd2o2_EDP3)R=(4fp^@CNbj-o0vz z7TDgix~pJbQc}(~CWEhvK-X$+o)db(481i`m&u~zF=h{y z;NCsynbDn4kMzT1N9uLD%#&Wu6;Q^~{k@)1#iLaTbJ%4vGw)7=|DrHciwT(;Qiae# z&dbdZ$e)JApoCP307%i|g^&u& zqqVnhyoZ2-k>E0lv@+tzEX}Y-kRlZ^E>X=UeC$4Hw=U)tP&Bs9!DDru_z&MAE|cj) z^0yS@+8*@$QIUIj$AVd^*U?uSsS<_&ERYg;k^68s45629oz~^ehWB!uuiQLi5-*db zG-S6XVi!BdR%KU=lOw@ac%seEA|~FiE&2Pn zspafIhzqYyUbtdRQ?9XDkcnf7%|f!uKimng-*A=ROcDkVx4H>X z$AQOtcImR&krJ-KIkN`8M*rHuO7w$#YJP3%P_sem+D;QSlo896nuv*LmF(y{;RTY% z0tlpJyeiHr%`bn+up0A+njte9P_h*$y@zTg%r4RT$ds!4Q)! zA?Y{Qp122!Pu;cm^&UV>f@YT(hB{UFhKm9rOs)vM5THa9_Ux>7TME`*6NxN%uLSye zpXPhxsTZH)o9oGae5BN0TkC8K(;C3t#%LD4l$7Ww2I}!it3xXjwsA$FI}&T@u_v~Z zqyUJ+5he*R?)+=aVjdjvjDb;z^z>n=|2dZgYRI{YW`j;|e@yJ|nVa>m>H3NHC}XZ~Ze-KiN1S>zjk0&KYacH60- ze{(6ZNi=mhL9{jHpRKHW!`Nh97=jD4R$*PaJmZ7p^%XNBc5~0FU+Y?jmrWB?C;Bvf zE$ggAIBHU6DqhpkWY7nau(-#T&xNvlf>X;!^oV&xAKTx5Ei*XH?XqcPRCb71ja{W* z+pPJjW=xIP)dw8_uP%$HU5aT;%O zsUAIfVTg&245-Q0X;3Z6@LYCdr>3}u;e9inNJ^VZdz)3z+3u?U*KfLud6^$ zI2-n!93}T|R(v5d0(K%MVMeJb!z1O=0^@f*D*nH{A=+Lj855El5$qYM>L|=AABYxt zrE{Rg25(D8$Ua8+>(b$EwZ(_1rY4M4eS*6y!e>e*>jHnp9#l@BjMSr#RreXM2bi$p z`J7i*$c^W7Gw6$)+T;(1lYF|obtZvQQ(}fjlEDQ}^6#($o*v~ppa1m1H9jqe39#Q;!fZI}pTkQ* zf>mAX5BA<<(tDD(nlnZQ>S;G)gfS6$S za(ueCY-E~wPAMV|p$(ajJ9I%Her{qK=VP5;$ls(OE)5i7_f>^Jh~3VkTLW$>ot^hG z=YR%XP^bjY4ieN0)MHt?&$Lyo_Vm-@4q#ANWJduB4)e-)MUKsYKR8}MUFkZYikDA+ z7CwW72`Jb0UVkkfe2DI1s3H9|JnA_@Fkb@o?VgfJ>Hvu!UX zr?V65(KUe4LFe=;Dgi8O{>!Kdw@qPvvca2j_lvO^lDD;vksV?}?vzo$K68}I5&_)S{kWEDyUiQsc$ol(nx8=%KHI-prykVb+sR1HUdKTG z*f_;PAr_B*s3b2ZTQpPNN_&y z@^3Ce;>Eo0_pi<{1PXh?7C*zXBe$FfA z{)oM-WAzgtRPKhlp@!})J4KY6S+^9EJXMsAWJD3TkFpqVQoCfM z&<#L+x1muhhx!qfWg}71WSm{6gcqPD|JWP4`Mqm~yH8}xq904Zrv}w>JVXd`VNtHYNMor}JHYB4pUhDJa<0;JsS%?h=cpJ$Aq`{u^|_ZJogs z_Ptw{*_?y8^OKBrlwIg|B_fy&r5@7<70{b6;&73Epu=qhvPqM$j^TX41F;Bb-qK>y zgl!=)iM=Nv(o9#dM7v33d-msPAthvi^J~kzH)Ls1xkx=pDW^L^xDD*|(LGS(_u~aD z*(%Xe@WhLf#KmM(0Y6RSh@&^lZqE9knJO1@@a`iyY^$?1m~MHT>g3EYCaNA>OQTo| z_K4b6GEwE=+=MgH>M-kM>Q?v71ZkiY&nR6(B;Kn(#s2=L0B=j1FjruF{Cs8lcKsf!>{?g&$eYGq?tLw__Lkl6mXLJ;oRvM19u-B6XYDDw01~3mARy7;V zjmN(L@1RmPMgrdDpCA$6?PY=F-EQp zDYI&cK7$3Ph&9TzYCU-wpwE1o?>Y;m!LX}ZbI7Zaf19*yA|GfPpOS8iyMOcHg3tTz;NFy1I{`5zEq1pE zQ1XG}m5vvn;Ihi^I;l}FT&d@{yp>xHRv?hatv60Y(MwuAnS8DOL52+W1+o8SWHi_S zT*PQO6z@w~#ncH6UNM=Iu=d1tMU=bH&Yg7913YVd==6|0j6(%gswvLhG%x}@$z@Od zl;-vo5C-B`3Aa=R_mh9C5C7-A`M=7?e%GExDL`v{3we`Nl@^56>-%_0CGcy}wxM3>((vqP+y z&;y+#GvY&oa1k1A%bKym(w=P;8esZ z@5TG^|ji90%uRm%64*Kz6kG~P~UwN9^ z#Er3P>k@0;Q@6R}X*CME_`}IZ9v*eI1bXp3{}D_wv^XVz=z-MamY{fdDDV{fE5=bd z<_a|Tk8X5#>ZX^;A02J_peZi(_%#6eli~8)jgiU;XMpahwf$e&Aw^BkU3z9nmociB zT#!Q++21Q#y9RQkb!-pr#h(8yES0EB>_f*OkkRr@lGWVK`;yxIjMo#$^z67D`oTd# z72=3zq3%ujK$Kk;z-POzEWUZ<@)N zYlmHRot?ST;d5HfB2&Dr!BJ@`BwGJT015?RWBFSQ_^seXs06@BUFQqfrgq>E08D@G z5yp`<*U|CE1MWBiIeq`Il^#KhIv3%Y^~X$e7Rm)I$;i;HdkK@=zPfFHP+^pFra^<9 zPRTU^x>8@92L2NKECiRsBjSMLf%k;|mk!$Bsz74tr3V@Z%gdmO$vY7-FA=zll~!pJ z|C2w?2yOZd{U@5$me}j62XK6m|ddqGA$OwydG(K(n>W zDOa%beejPhQ+a}lx30p&-+IlE(mS?xSpkMu-Xu7YL3dk1O?}FrOz6L8Q_g6o*O-iK zxhAdw{9K`B{NgyE=Ad8%F(;(?D<||?Yx}?EgiiRCI7W@;Iy<1h%NG+^AXaY}W{y>* z$1`s0|E?BRHtP?3Wx6+Vj$PflfoWX6Z4~!Zshs%8e%?;#9y{df zIr12NMP05dbMvhv-JvthC0K)-e>Kp{0=gf4Q6dJ=7blu-B~`N%h`w?o!8ws&8FHAE z1o?(*e*l`-c+Yy9E!_)rrR={Wk;DAZs}S42jrKR!#2~>>y0_@XAlL8irhpvwE&hKS zjOr8kH5p9mpNu+7z`;xZbKQ+q2<(5$Uh;|@7W%y63l3QZ5vN`!^z`x)1(0EtRG0bO zQ=Qu#Mnw)ATX71EV?1F&@H!U9!`<&GUb?SfOJ{0fjzD2w}Nr^*036ZoXVv7Amui9fgK7 zV)cDU^kTIoxfL6ZHiS$rzY6cBc_9r&Y=l(5jo-cHgcJ>{H|$KgilQTDao=FbM#x4CP%@%DS=^_EfG;4p8hf16PB;eF`FnvAN; zFAs}O_Bq`7G_R@wbqi+)7%Q~X zY7`tR8Vine*7lAXTRz@{D=5{ApOO?Y7vBw3!N(tN#`um-taA>ELV4@XKDslE-!*j0 zu|q^eaYMjBO2!jd`5A$W(gdr2c}kr#}=e@wp5VpE}m*vVFeQ z)GHo%3wu|b;r}%hvY+RKUiHMS%ipVN=_{}_b(hi<&p`lLD%2%!8V{mgsWL(rL&0?_ zmcOA4N1&Y3f$0sejy4vh+W^vZvAKH6)Q=o*DN5m7yQGA5lOJoj#?Mrk1rggMVdaf@3{s+c69@pm+T^A=~BlutCw%%J8v*SMYMZBz9l? zk_RPKD<2kF-KVHv1L|iVU5S4BR94Buc2=r(>Zmmt!yg?J#ussIJhp3=%3GwMx|PdC zDGJ!69O-N`o%+(|6at`H^RJlB;siW@Afr31POt4$ydBJ$OCd=QkoMJ+U{-V%jBndGiu`CIWi$^_^QE z_h#f4tfl3rxpL_6EQInBh%sh4)!PbLCRPgAk$OxDIl-F-&Pu0(gZ_~`>3FOV2Mc)Sb zO6CRa!>7V)GzXIy-m2);k_BIV;AfD;dY+Nx#}ma+r>JDD#}Mf+_Q}^Kxot*)wrpYf z#Zz(`c%@}n=lUM*sc|gStV0+h(%{o_Vt(RLNZF?mh@{|_1&M1``fd8`5z*CTIOgF} z#no9}+@Y?NzLK8G`>%bS#b>!YHO>5`d8p{qqHCS6K-~Gl4}KRW7LOrKf2ydfpjYNT z+T@bjkFGtr_=%H|cDd(ULs>$)^`>P0kKX`jXeDvN0w`O{PYXO@HZ2iI$)LwKA}7aK zUZ2_W<3se2V_h>UjzvWsg$CcJgzZW1EXVUEV=uV2p02FJ(*c|=L?nk`g~FV(?P5hq zBE#>Bcii1jW~$4v0OHYzXw$p*;k|vp4OGb85MuHdv?re`)~vy(w2Ec zb$*^aBOtcPc`ziAr<4L!KuM|g2v)5dK+^L75i3)5y6K1#Gw~8oJl^#opxKqIksba9 z20zaTdEN=!lL#LvX&ut}9gJ}qx@&C70r?5q9-h0UF)6-nAQ|<(oV>rp78mq}+mTl4 z*?Do11YI8cRgosQ2r_+VVUpOE0U}*dcpfwR{aOv{%2Ozeox?%Pj-e;eWc`BzsO2DK8j=U^j8$6(zk@Sv%QWxn zot2kY32*6|(aSI#aGka(VeR!RdeR>z6qaq!)UG}WoNu^cq%CNUnK(3rp^sbdr!J(a z=_1kA^K-a&b;im6%PP5?yZd|~3sWI(od9HuTkZRbeFq2GyD`V|c#{MD3M#m6z0Nc( zfRpYcjSpI0ZOUCAg@<4Fuf`Z-Hq;8-jzH4yr)?~&9x1&;O6ELfVzIShH$EP{yVa~; zSiLOT3L&426O9vk8K6>Qihn&++hBFQ*K@c%&v-tfH;J?hI-CyZ6qMAasx^{+v0YPQ z19utE*LGfFW|+PZQVfyzzPI}XZ>OI!rBu5?FjZWK!vuDaPnJ*DqCXEE(U|&`FG0SL zPf91fWN?^7_eA`226$DT=579Vi7^A?YiUI5R;tVz=2xGo(w>EaqxowV4;;i!A$#%8 z$%<@c+)sQC>anwdQ%eyK5#BiP29-WwNG4^eM7T%p?f0L6h+3wir`v0Aa!H$B;Mt8>S{DQGzw|+ zbF=~ygZ1TXV55$^g{-Tsfy3ucU$0~)z!QFzO3?vFb4ho%WdrgKD_2Djle=}%xSev5 z{Gg$oo4RWXCPdFlnbq(pwYyYsWOxz5Su9O8$ro<-bo{B2tT_7bxWL~c`7bVzVe;(kt#VI431y^L|8D{9i6*}5oB{rE3c!sfgG6{;vjveo_dzIL?B$x(lrG~DN!&b-;s%24*b*6ySjBg3(na@~&(y?p9nE~qe zTCpmlHGg%kTJ{J}w@Y6DUzrWkmH!27$D?Q5hw~RO6h-d=e{CWZ#q(m9>tXCAbwo9| zL@s+E#}Ih{vyCL-69W?Tp{ZE+QgaT{B!+)*`~&Iw#mkP-e7b}%OSB9}niM=qX3P z0#du~v=E2My(Ia+9HT^k7xBO4z5aRHO0_yd=Z-YmFQ2KMKt}-m;RirjQR8`)yjk%7 E0q&R<=l}o! literal 0 HcmV?d00001 From 6cc23745aae8775d741555cfba797a4e054d8d18 Mon Sep 17 00:00:00 2001 From: Anthony Marakis Date: Sun, 16 Jul 2017 18:30:35 +0300 Subject: [PATCH 4/5] Update nlp.ipynb --- nlp.ipynb | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 229 insertions(+), 2 deletions(-) diff --git a/nlp.ipynb b/nlp.ipynb index 5600a308e..64d5c7082 100644 --- a/nlp.ipynb +++ b/nlp.ipynb @@ -20,7 +20,7 @@ "outputs": [], "source": [ "import nlp\n", - "from nlp import Page, HITS" + "from nlp import Page, HITS, Lexicon, Rules, Grammar" ] }, { @@ -32,6 +32,7 @@ "## CONTENTS\n", "\n", "* Overview\n", + "* Languages\n", "* HITS\n", "* Question Answering" ] @@ -45,6 +46,232 @@ "`TODO...`" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LANGUAGES\n", + "\n", + "Languages can be represented by a set of grammar rules over a lexicon of words. Different languages can be represented by different types of grammar, but in Natural Language Processing we are mainly interested in context-free grammars.\n", + "\n", + "### Context-Free Grammars\n", + "\n", + "A lot of natural and programming languages can be represented by a **Context-Free Grammar (CFG)**. A CFG is a grammar that has a single non-terminal symbol on the left-hand side. That means a non-terminal can be replaced by the right-hand side of the rule regardless of context. An example of a CFG:\n", + "\n", + "```\n", + "S -> aSb | e\n", + "```\n", + "\n", + "That means `S` can be replaced by either `aSb` or `e` (with `e` we denote the empty string). The lexicon of the language is comprised of the terminals `a` and `b`, while with `S` we denote the non-terminal symbol. In general, non-terminals are capitalized while terminals are not, and we usually name the starting non-terminal `S`. The language generated by the above grammar is the language anbn for n greater or equal than 1." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Probabilistic Context-Free Grammar\n", + "\n", + "While a simple CFG can be very useful, we might want to know the chance of each rule occuring. Above, we do not know if `S` is more likely to be replaced by `aSb` or `e`. **Probabilistic Context-Free Grammars (PCFG)** are built to fill exactly that need. Each rule has a probability, given in brackets, and the probabilities of a rule sum up to 1:\n", + "\n", + "```\n", + "S -> aSb [0.7] | e [0.3]\n", + "```\n", + "\n", + "Now we know it is more likely for `S` to be replaced by `aSb` than by `e`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Lexicon\n", + "\n", + "The lexicon of a language is defined as a list of allowable words. These words are grouped into the usual classes: `verbs`, `nouns`, `adjectives`, `adverbs`, `pronouns`, `names`, `articles`, `prepositions` and `conjuctions`. For the first five classes it is impossible to list all words, since words are continuously being added in the classes. Recently \"google\" was added to the list of verbs, and words like that will continue to pop up and get added to the lists. For that reason, these first five categories are called **open classes**. The rest of the categories have much fewer words and much less development. While words like \"thou\" were commonly used in the past but have declined almost completely in usage, most changes take many decades or centuries to manifest, so we can safely assume the categories will remain static for the foreseeable future. Thus, these categories are called **closed classes**.\n", + "\n", + "An example lexicon for a PCFG (note that other classes can also be used according to the language, like `digits`, or `RelPro` for relative pronoun):\n", + "\n", + "```\n", + "Verb -> is [0.3] | say [0.1] | are [0.1] | ...\n", + "Noun -> robot [0.1] | sheep [0.05] | fence [0.05] | ...\n", + "Adjective -> good [0.1] | new [0.1] | sad [0.05] | ...\n", + "Adverb -> here [0.1] | lightly [0.05] | now [0.05] | ...\n", + "Pronoun -> me [0.1] | you [0.1] | he [0.05] | ...\n", + "RelPro -> that [0.4] | who [0.2] | which [0.2] | ...\n", + "Name -> john [0.05] | mary [0.05] | peter [0.01] | ...\n", + "Article -> the [0.35] | a [0.25] | an [0.025] | ...\n", + "Preposition -> to [0.25] | in [0.2] | at [0.1] | ...\n", + "Conjuction -> and [0.5] | or [0.2] | but [0.2] | ...\n", + "Digit -> 1 [0.3] | 2 [0.2] | 0 [0.2] | ...\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Grammar\n", + "\n", + "With grammars we combine words from the lexicon into valid phrases. A grammar is comprised of **grammar rules**. Each rule transforms the left-hand side of the rule into the right-hand side. For example, `A -> B` means that `A` transforms into `B`. Let's build a grammar for the language we started building with the lexicon. We will use a PCFG.\n", + "\n", + "```\n", + "S -> NP VP [0.9] | S Conjuction S [0.1]\n", + "\n", + "NP -> Pronoun [0.3] | Name [0.1] | Noun [0.1] | Article Noun [0.25] |\n", + " Article Adjs Noun [0.05] | Digit [0.05] | NP PP [0.1] |\n", + " NP RelClause [0.05]\n", + "\n", + "VP -> Verb [0.4] | VP NP [0.35] | VP Adjective [0.05] | VP PP [0.1]\n", + " VP Adverb [0.1]\n", + "\n", + "Adjs -> Adjective [0.8] | Adjective Adjs [0.2]\n", + "\n", + "PP -> Preposition NP [1.0]\n", + "\n", + "RelClause -> RelPro VP [1.0]\n", + "```\n", + "\n", + "Some valid phrases the grammar produces: \"`mary is sad`\", \"`you are a robot`\" and \"`she likes mary and a good fence`\".\n", + "\n", + "What if we wanted to check if the phrase \"`mary is sad`\" is actually a valid sentence? We can use a **parse tree** to constructively prove that a string of words is a valid phrase in the given language and even calculate the probability of the generation of the sentence.\n", + "\n", + "![parse_tree](images/parse_tree.png)\n", + "\n", + "The probability of the whole tree can be calculated by multiplying the probabilities of each individual rule transormation: `0.9 * 0.1 * 0.05 * 0.05 * 0.4 * 0.05 * 0.3 = 0.00000135`.\n", + "\n", + "To conserve space, we can also write the tree in linear form:\n", + "\n", + "[S [NP [Name **mary**]] [VP [VP [Verb **is**]] [Adjective **sad**]]]\n", + "\n", + "Unfortunately, the current grammar **overgenerates**, that is, it creates sentences that are not grammatically correct (according to the English language), like \"`the fence are john which say`\". It also **undergenerates**, which means there are valid sentences it does not generate, like \"`he believes mary is sad`\"." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Implementation\n", + "\n", + "In the module we have implemented a `Lexicon` and a `Rules` function, which we can combine to create a `Grammar` object.\n", + "\n", + "Execute the cells below to view the implemenations:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%psource Lexicon" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%psource Rules" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "collapsed": true + }, + "outputs": [], + "source": [ + "%psource Grammar" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's build a lexicon and a grammar for the above language:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Lexicon {'Verb': ['is', 'say', 'are'], 'RelPro': ['that', 'who', 'which'], 'Noun': ['robot', 'sheep', 'fence'], 'Article': ['the', 'a', 'an'], 'Conjuction': ['and', 'or', 'but'], 'Pronoun': ['me', 'you', 'he'], 'Adjective': ['good', 'new', 'sad'], 'Name': ['john', 'mary', 'peter'], 'Digit': ['1', '2', '0'], 'Preposition': ['to', 'in', 'at'], 'Adverb': ['here', 'lightly', 'now']}\n", + "\n", + "Rules: {'S': [['NP', 'VP'], ['S', 'Conjuction', 'S']], 'VP': [['Verb'], ['VP', 'NP'], ['VP', 'Adjective'], ['VP', 'PP'], ['VP', 'Adverb']], 'PP': [['Preposition', 'NP']], 'RelClause': [['RelPro', 'VP']], 'Adjs': [['Adjective'], ['Adjective', 'Adjs']], 'NP': [['Pronoun'], ['Name'], ['Noun'], ['Article', 'Noun'], ['Article', 'Adjs', 'Noun'], ['Digit'], ['NP', 'PP'], ['NP', 'RelClause']]}\n" + ] + } + ], + "source": [ + "lexicon = Lexicon(\n", + " Verb=\"is | say | are\",\n", + " Noun=\"robot | sheep | fence\",\n", + " Adjective=\"good | new | sad\",\n", + " Adverb=\"here | lightly | now\",\n", + " Pronoun=\"me | you | he\",\n", + " RelPro=\"that | who | which\",\n", + " Name=\"john | mary | peter\",\n", + " Article=\"the | a | an\",\n", + " Preposition=\"to | in | at\",\n", + " Conjuction=\"and | or | but\",\n", + " Digit=\"1 | 2 | 0\"\n", + ")\n", + "\n", + "print(\"Lexicon\", lexicon)\n", + "\n", + "rules = Rules(\n", + " S=\"NP VP | S Conjuction S\",\n", + " NP=\"Pronoun | Name | Noun | Article Noun | Article Adjs Noun | Digit | NP PP | NP RelClause\",\n", + " VP=\"Verb | VP NP | VP Adjective | VP PP | VP Adverb\",\n", + " Adjs=\"Adjective | Adjective Adjs\",\n", + " PP=\"Preposition NP\",\n", + " RelClause=\"RelPro VP\"\n", + ")\n", + "\n", + "print(\"\\nRules:\", rules)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Both the functions return a dictionary with keys the left-hand side of the rules. For the lexicon, the values are the terminals for each left-hand side non-terminal, while for the rules the values are the right-hand sides as lists.\n", + "\n", + "We can now use the variables `lexicon` and `rules` to build a grammar. After we've done so, we can find the transformations of a non-terminal (the `Noun`, `Verb` and the other basic classes do **not** count as proper non-terminals in the implementation). We can also check if a word is in a particular class." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "How can we rewrite 'VP'? [['Verb'], ['VP', 'NP'], ['VP', 'Adjective'], ['VP', 'PP'], ['VP', 'Adverb']]\n", + "Is 'the' an article? True\n", + "Is 'here' a noun? False\n" + ] + } + ], + "source": [ + "grammar = Grammar(\"A Simple Grammar\", rules, lexicon)\n", + "\n", + "print(\"How can we rewrite 'VP'?\", grammar.rewrites_for('VP'))\n", + "print(\"Is 'the' an article?\", grammar.isa('the', 'Article'))\n", + "print(\"Is 'here' a noun?\", grammar.isa('here', 'Noun'))" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -245,7 +472,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2+" + "version": "3.5.3" } }, "nbformat": 4, From 4c03e8c39426a3f22c764ce58a7c66197edcba2c Mon Sep 17 00:00:00 2001 From: Anthony Marakis Date: Mon, 17 Jul 2017 17:07:56 +0300 Subject: [PATCH 5/5] add generate_random --- nlp.ipynb | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/nlp.ipynb b/nlp.ipynb index 64d5c7082..12e00ba15 100644 --- a/nlp.ipynb +++ b/nlp.ipynb @@ -198,16 +198,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Lexicon {'Verb': ['is', 'say', 'are'], 'RelPro': ['that', 'who', 'which'], 'Noun': ['robot', 'sheep', 'fence'], 'Article': ['the', 'a', 'an'], 'Conjuction': ['and', 'or', 'but'], 'Pronoun': ['me', 'you', 'he'], 'Adjective': ['good', 'new', 'sad'], 'Name': ['john', 'mary', 'peter'], 'Digit': ['1', '2', '0'], 'Preposition': ['to', 'in', 'at'], 'Adverb': ['here', 'lightly', 'now']}\n", + "Lexicon {'Article': ['the', 'a', 'an'], 'Adverb': ['here', 'lightly', 'now'], 'Digit': ['1', '2', '0'], 'Pronoun': ['me', 'you', 'he'], 'Name': ['john', 'mary', 'peter'], 'Adjective': ['good', 'new', 'sad'], 'Conjuction': ['and', 'or', 'but'], 'Preposition': ['to', 'in', 'at'], 'RelPro': ['that', 'who', 'which'], 'Verb': ['is', 'say', 'are'], 'Noun': ['robot', 'sheep', 'fence']}\n", "\n", - "Rules: {'S': [['NP', 'VP'], ['S', 'Conjuction', 'S']], 'VP': [['Verb'], ['VP', 'NP'], ['VP', 'Adjective'], ['VP', 'PP'], ['VP', 'Adverb']], 'PP': [['Preposition', 'NP']], 'RelClause': [['RelPro', 'VP']], 'Adjs': [['Adjective'], ['Adjective', 'Adjs']], 'NP': [['Pronoun'], ['Name'], ['Noun'], ['Article', 'Noun'], ['Article', 'Adjs', 'Noun'], ['Digit'], ['NP', 'PP'], ['NP', 'RelClause']]}\n" + "Rules: {'Adjs': [['Adjective'], ['Adjective', 'Adjs']], 'PP': [['Preposition', 'NP']], 'RelClause': [['RelPro', 'VP']], 'VP': [['Verb'], ['VP', 'NP'], ['VP', 'Adjective'], ['VP', 'PP'], ['VP', 'Adverb']], 'NP': [['Pronoun'], ['Name'], ['Noun'], ['Article', 'Noun'], ['Article', 'Adjs', 'Noun'], ['Digit'], ['NP', 'PP'], ['NP', 'RelClause']], 'S': [['NP', 'VP'], ['S', 'Conjuction', 'S']]}\n" ] } ], @@ -251,7 +251,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -272,6 +272,35 @@ "print(\"Is 'here' a noun?\", grammar.isa('here', 'Noun'))" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Finally, we can generate random phrases using our grammar. Most of them will be complete gibberish, falling under the overgenerated phrases of the grammar. That goes to show that in the grammar the valid phrases are much fewer than the overgenerated ones." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'a robot is to a robot sad but robot say you 0 in me in a robot at the sheep at 1 good an fence in sheep in me that are in john new lightly lightly here a new good new robot lightly new in sheep lightly'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from nlp import generate_random\n", + "\n", + "generate_random(grammar)" + ] + }, { "cell_type": "markdown", "metadata": {},