Thanks to visit codestin.com
Credit goes to www.scribd.com

Scribd
Upload
140 views8 pages

Import: "../information Retrieval/" "Doc .TXT" ' '

This document discusses implementing basic information retrieval operations like AND, OR, and NOT on an inverted index. It contains code to: 1) Create an inverted index from text documents and list of words. 2) Define functions for AND, OR, and NOT operations on the inverted index that take query words as input and return the result postings lists. 3) Demonstrate examples of queries like "home and july", "sales or forecasts", and "sales-july". 4) Discuss an optimized intersect algorithm using skip pointers to reduce the number of comparisons.

Uploaded by

Gurprem Singh
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as PDF, TXT or read online on Scribd
140 views8 pages

Import: "../information Retrieval/" "Doc .TXT" ' '

This document discusses implementing basic information retrieval operations like AND, OR, and NOT on an inverted index. It contains code to: 1) Create an inverted index from text documents and list of words. 2) Define functions for AND, OR, and NOT operations on the inverted index that take query words as input and return the result postings lists. 3) Demonstrate examples of queries like "home and july", "sales or forecasts", and "sales-july". 4) Discuss an optimized intersect algorithm using skip pointers to reduce the number of comparisons.

Uploaded by

Gurprem Singh
Copyright
© © All Rights Reserved
We take content rights seriously. If you suspect this is your content, claim it here.
Available Formats
Download as PDF, TXT or read online on Scribd
You are on page 1/ 8

IR Assignment 1 21/08/18, 10(35 PM

In [1]: import glob


docs=[]
content=[]
doc_list=[]
i=0
directory= "../Information Retrieval/"
for files in glob.iglob(directory + "doc*.txt"):
infile = open(files)
a=infile.readline().split(' ')
doc_list.append(i)
i+=1
content+=a
print(a)
docs.append(a)
#for k in range (0,len(a)):
#print(a[0])
infile.close()
#The following printout is the corresponding doc no i.e. doc 0,1,2,3.

['july', 'new', 'home', 'sales', 'rise']


['new', 'home', 'sales', 'top', 'forecasts']
['increase', 'in', 'home', 'sales', 'in', 'july']
['home', 'sales', 'rise', 'in', 'july']

In [2]: #content contains all the words of the corpus


print(content)

['july', 'new', 'home', 'sales', 'rise', 'new', 'home', 'sales', 'to


p', 'forecasts', 'increase', 'in', 'home', 'sales', 'in', 'july', 'h
ome', 'sales', 'rise', 'in', 'july']

In [3]: #list of lists i.e. words occuring in different documents


print(docs)

[['july', 'new', 'home', 'sales', 'rise'], ['new', 'home', 'sales',


'top', 'forecasts'], ['increase', 'in', 'home', 'sales', 'in', 'july
'], ['home', 'sales', 'rise', 'in', 'july']]

In [4]: print(doc_list)

[0, 1, 2, 3]

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 1 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [5]: #dictionary for inverted index


# the data point at index 0 for every word in inverted_index is the freque
#e.g. in('rise', [2, 0, 3])~ 2 is frequency
#and 0,3 are the documents in which it occurs.
inv_indx = {i:[] for i in content}
for word in content:
if not inv_indx[word]:
cnt=0
for i in range(len(docs)):
if word in docs[i]:
inv_indx[word].append(i)
cnt+=1
inv_indx[word].insert(0,cnt)

In [6]: #print inverted index


for k,v in inv_indx.items():
print(k,v)
This is the frequency
('in', [2, 2, 3]) This is postings list
('top', [1, 1])
('rise', [2, 0, 3])
('sales', [4, 0, 1, 2, 3])
('forecasts', [1, 1])
('increase', [1, 2])
('home', [4, 0, 1, 2, 3])
('new', [2, 0, 1])
('july', [3, 0, 2, 3])

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 2 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [7]: #AND - merge algorithm


def intersect(inv_indx,w1,w2):
answer=[]
if w1 in inv_indx:
p1=inv_indx[w1]
print(p1)
else: print(w1+" not present in any doc")
if w2 in inv_indx:
p2=inv_indx[w2]
print(p2)
else: print(w2+" not present in any doc")
i=1
j=1
num=min(len(p1),len(p2))
num=num-1
while i<=num or j<= num:
print(p1[i],p2[j])
if p1[i]==p2[j]:
answer.append(p1[i])
i+=1
j+=1
elif p1[i]<p2[j]:
i+=1
else: j+=1
return answer

In [8]: #question "home and july"


ans=intersect(inv_indx,'home','july')
print(ans)

[4, 0, 1, 2, 3]
[3, 0, 2, 3]
(0, 0)
(1, 2)
(2, 2)
(3, 3)
[0, 2, 3]

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 3 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [9]: def OR (w1,w2):


answer=[]
if w1 in inv_indx:
p1=inv_indx[w1]
print(p1)
else: print(w1+" not present in any doc")
if w2 in inv_indx:
p2=inv_indx[w2]
print(p2)
else: print(w2+" not present in any doc")
i=1
j=1
num=min(len(p1),len(p2))
num=num-1
while i<=num or j<= num:
print(p1[i],p2[j])
if p1[i]==p2[j]:
answer.append(p1[i])
i+=1
j+=1
elif p1[i]<p2[j]:
answer.append(p1[i])
i+=1
else: answer.append(p2[j]);j+=1

if i<j: answer.append(p2[j:])
else: answer.append(p1[i:])
return answer

In [10]: #question"sales or forecasts


ans=OR('sales','forecasts')
print(ans)

[4, 0, 1, 2, 3]
[1, 1]
(0, 1)
(1, 1)
[0, 1, [2, 3]]

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 4 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [11]: def minus (w1,w2):


answer=[]
if w1 in inv_indx:
p1=inv_indx[w1]
print(p1)
else: print(w1+" not present in any doc")
if w2 in inv_indx:
p2=inv_indx[w2]
print(p2)
else: print(w2+" not present in any doc")
i=1
j=1
num=min(len(p1),len(p2))
num=num-1
while i<=num or j<= num:
print(p1[i],p2[j])
if p1[i]==p2[j]:
i+=1
j+=1
elif p1[i]<p2[j]:
answer.append(p1[i])
i+=1
else: answer.append(p2[j]);j+=1

answer.append(p1[i:])
return answer

In [12]: #question "sales-july"


ans=minus('sales','july')
print(ans)

[4, 0, 1, 2, 3]
[3, 0, 2, 3]
(0, 0)
(1, 2)
(2, 2)
(3, 3)
[1, []]

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 5 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [14]: def NOT (w1):


answer=[]
if w1 in inv_indx:
p1=inv_indx[w1]
print(p1)
else: print(w1+" not present in any doc")
num=len(p1)-1
i=1
temp=doc_list
while i<= num:
if p1[i] in doc_list:
temp.remove(p1[i])
i+=1
answer=temp
return answer

In [15]: #question not july


print(NOT('july'))

[3, 0, 2, 3]
[1]

In [16]: l1=[4,6,10,12,14,16,18,20,22,32,47,81,120,122,157,180]
l2=[47]

In [17]: #AND - merge algorithm


import math
def intersect_skip(p1,p2):
answer=[]
i=0
j=0
num=max(len(p1),len(p2))
print(num)
skip=int(math.sqrt(num))
while i<=num:
print(p1[i],p2[j])
if p1[i]==p2[j]:
answer.append(i)
i+=1
return answer
elif p1[i]<p2[j]:
i+=skip
else: i-=1
return answer

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 6 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [18]: print(intersect_skip(l1,l2))
#the number of comparisons here with skip pointers is 6
#due to the fact that lists are sorted and we can easily skip
#and adjust the pointer in accordance with the value being searched.

16
(4, 47)
(14, 47)
(22, 47)
(120, 47)
(81, 47)
(47, 47)
[10] The index of first occurence

In [19]: #AND - merge algorithm


import math
def intersect_normal(p1,p2):
answer=[]
i=0
j=0
num=max(len(p1),len(p2))
print(num)
skip=int(math.sqrt(num))
while i<=num:
print(p1[i],p2[j])
if p1[i]==p2[j]:
answer.append(i)
i+=1
return answer
elif p1[i]<p2[j]:
i+=1
else: i-=1
return answer

In [20]: print(intersect_normal(l1,l2))
#here 11 comparisons needed since we need to move in linear fashion
# no use of the soretd lists is made of

16
(4, 47)
(6, 47)
(10, 47)
(12, 47)
(14, 47)
(16, 47)
(18, 47)
(20, 47)
(22, 47)
(32, 47)
(47, 47)
[10]

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 7 of 8
IR Assignment 1 21/08/18, 10(35 PM

In [ ]:

http://localhost:8888/notebooks/mlworkshop/Information%20Retrieval/IR%20Assignment%201.ipynb# Page 8 of 8

You might also like