Basic Text Processing operation on text document.
for line in open("data.txt"):
for word in line.split():
if word.endswith("ing"):
print(word)
print(len(word))
import re
data="The biggest 5 Animals are 1.Elephant 2.Girrafe 3.Tiger 4.Lion 5.Cheetah"
result=re.sub("\d+",'',data)
print(result)
def punctuations(data):
text=data
text=text.replace("'nt"," not")
text=text.replace("'s"," is")
text=text.replace("'re"," are")
text=text.replace("'ll"," will")
return text
s="How's my team doin,you're supposed to be not losing"
returned_data=punctuations(s)
print(returned_data)
Tokenization, Stemming
import nltk
nltk.download()
import nltk
data="Welcome to TIMSCDR!!"
tokens=nltk.sent_tokenize(data)
print(tokens)
tokens=nltk.word_tokenize(data)
print(tokens)
from nltk.stem import PorterStemmer
port_stemmer=PorterStemmer()
print(port_stemmer.stem("Liked"))
data=["liked","liking","likes","killing","killed"]
for words in data:
print(words," :",port_stemmer.stem(words))
from nltk.stem import WordNetLemmatizer
lemmati=WordNetLemmatizer()
print("Socks :",lemmati.lemmatize("socks"))
print("corpora :",lemmati.lemmatize("corpora"))
print("better :",lemmati.lemmatize("better",pos="a"))
Removal of Stopwords
from nltk.corpus import stopwords
nltk.download('stopwords')
stopwords=set(stopwords.words('english'))
print(stopwords)
from nltk.tokenize import sent_tokenize,word_tokenize
from nltk.corpus import stopwords
stopWords=set(stopwords.words('english'))
data="All work and no play.All work and no play makes jack a dull boy"
tokens=nltk.word_tokenize(data)
filtered_Data=[]
for w in tokens:
if w not in stopWords:
filtered_Data.append(w)
print(filtered_Data)
Implementation of POS Tag
1.
import spacy
nlp=spacy.load("en_core_web_sm")
doc=nlp("Don't be afraid to give up the good to go for the great")
POS_count= doc.count_by(spacy.attrs.POS)
print(POS_count)
for k,v in sorted(POS_count.items()):
print(f'{k}. {doc.vocab[k].text} : {v}')
2.
import spacy
from spacy import displacy
nlp=spacy.load("en_core_web_sm")
doc=nlp("This is my School")
options={'color':'red','bg':'blue','compact':'True','distance':100}
displacy.render(doc,style="dep",options=options)
3.
import nltk
from nltk.tag import DefaultTagger
text="The way to get started is to quit talking"
tokens=nltk.word_tokenize(text)
tagging=DefaultTagger("Ad")
print(tagging.tag(tokens))
4.
import nltk
import nltk
sentence= "the little yellow dog barked at the car"
#Tokenization
tokens= nltk.word_tokenize(sentence)
#POs Tagging
tag=nltk.pos_tag(tokens)
phrase= "NP: {<DT>?<JJ>*<NN>}"
cp=nltk.RegexpParser(phrase)
result=cp.parse(tag)
print(result)
TF-IDF implementation
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
text = ["I love writing code in Python. I love Python code",
"I hate writing code in Java. I hate Java code"]
df = pd.DataFrame({'review':['review1','review2'],'text':text})
cv = CountVectorizer(stop_words='english')
cv_matrix = cv.fit_transform(df['text'])
df_dtm=pd.DataFrame(cv_matrix.toarray(),index=df['review'].values,
columns=cv.get_feature_names_out())
df_dtm
pip install sklearn
import pandas as pd
from sklearn.feature_extraction.text import TfidfVectorizer
text = ["I love writing code in python.I love python code",
"I hate writing code in java.i hate java code"]
df = pd.DataFrame({'review':['review1','review2'],'text':text})
tfidf = TfidfVectorizer(stop_words='english',norm=None)
tfidf_matrix = tfidf.fit_transform(df['text'])
output = pd.DataFrame(tfidf_matrix.toarray(),index=df['review'],
columns=tfidf.get_feature_names_out())
output
Practical 4: Creating and comparing different text representations
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
import numpy as np
def compare_text_representations(texts):
# Bag of Words (BoW) representation
bow_vectorizer = CountVectorizer()
bow_matrix = bow_vectorizer.fit_transform(texts)
# TF-IDF representation
tfidf_vectorizer = TfidfVectorizer()
tfidf_matrix = tfidf_vectorizer.fit_transform(texts)
# Compare similarities
bow_similarity = cosine_similarity(bow_matrix[0], bow_matrix[1])
tfidf_similarity = cosine_similarity(tfidf_matrix[0], tfidf_matrix[1])
print("BoW vectors:")
print(bow_matrix.toarray())
print("\nTF-IDF vectors:")
print(tfidf_matrix.toarray())
print(f"\nBoW Cosine Similarity: {bow_similarity[0][0]:.4f}")
print(f"TF-IDF Cosine Similarity: {tfidf_similarity[0][0]:.4f}")
# Example usage
texts = [
"The cat sat on the mat",
"The dog sat on the log"
]
compare_text_representations(texts)
Training and using word embeddings
from gensim.models import Word2Vec
from nltk.tokenize import word_tokenize
import nltk
nltk.download('punkt')
def train_word_embeddings(sentences):
# Tokenize sentences
tokenized_sentences = [word_tokenize(sentence.lower()) for sentence in sentences]
# Train Word2Vec model
model = Word2Vec(sentences=tokenized_sentences, vector_size=100,
window=5, min_count=1, workers=4)
return model
def use_word_embeddings(model, word, top_n=5):
try:
similar_words = model.wv.most_similar(word, topn=top_n)
print(f"Words most similar to '{word}':")
for w, score in similar_words:
print(f"{w}: {score:.4f}")
except KeyError:
print(f"'{word}' not in vocabulary")
# Example usage
sentences = [
"The quick brown fox jumps over the lazy dog",
"A fox is a cunning animal",
"The dog barks at night",
"Foxes and dogs are different species"
]
model = train_word_embeddings(sentences)
use_word_embeddings(model, "fox")
Implement N gram Language model
1.
import nltk
from nltk.util import ngrams
text="The little boy ran away"
Ngram= ngrams(sequence=nltk.word_tokenize(text),n=3)
for gram in Ngram:
print(gram)
2.
import nltk
from nltk import bigrams, trigrams
from collections import defaultdict
import random
nltk.download('punkt')
def build_language_model(text, n=2):
words = nltk.word_tokenize(text.lower())
if n == 2:
pairs = list(bigrams(words))
elif n == 3:
pairs = list(trigrams(words))
else:
raise ValueError("n must be 2 or 3")
model = defaultdict(lambda: defaultdict(int))
for pair in pairs:
if n == 2:
model[pair[0]][pair[1]] += 1
else:
model[(pair[0], pair[1])][pair[2]] += 1
return model
def generate_text(model, num_words=20, start_word=None, n=2):
if start_word is None:
start_word = random.choice(list(model.keys()))
words = [start_word] if n == 2 else list(start_word)
for _ in range(num_words - n + 1):
if n == 2:
last_word = words[-1]
next_word = max(model[last_word], key=model[last_word].get)
else:
last_words = tuple(words[-2:])
next_word = max(model[last_words], key=model[last_words].get)
words.append(next_word)
return ' '.join(words)
# Example usage
text = """
The cat sat on the mat. The dog ran in the park.
Cats like to play with toys. Dogs enjoy chasing balls.
"""
bigram_model = build_language_model(text, n=2)
trigram_model = build_language_model(text, n=3)
print("Generated text (bigram model):")
print(generate_text(bigram_model, num_words=15, start_word="the", n=2))
print("\nGenerated text (trigram model):")
print(generate_text(trigram_model, num_words=15, start_word=("the", "cat"),
n=3))
Implementation text classification using Naïve Bayes
import pandas as pd
df = pd.read_csv('twitter_training.csv',header=None,usecols=[2,3])
df.head()
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
#Rename columns
df.columns=['Sentiment','Text']
#Removes the missing values
df= df.dropna()
#encode label
le= LabelEncoder()
df['Sentiment']=le.fit_transform(df['Sentiment'])
df.head()
X=list(df['Text'])
y=list(df['Sentiment'])
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.25,
random_state=42)
from sklearn.feature_extraction.text import CountVectorizer
cv=CountVectorizer(analyzer='word',ngram_range=(1,1),stop_words='english')
X_train_cv = cv.fit_transform(X_train)
X_test_cv = cv.transform(X_test)
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import f1_score
import numpy as np
#train Naive Bayes Classifier
clf=MultinomialNB()
clf.fit(X_train_cv, y_train)
y_pred=clf.predict(X_test_cv)
score=f1_score(y_test,y_pred, average="micro")
print('F-1 score : {}'.format(np.round(score,4)))
for i in range(1,11):
cv=CountVectorizer(analyzer='word',ngram_range=(1,i),stop_words='english')
#creating BOG
X_train_cv=cv.fit_transform(X_train)
X_test_cv=cv.transform(X_test)
#training of classifiers
clf2=MultinomialNB()
clf2.fit(X_train_cv,y_train)
y_pred=clf2.predict(X_test_cv)
score=f1_score(y_test,y_pred,average="micro")
print('F1-Score: {}'.format(np.round(score,4)))
Implementing a text classifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
def train_text_classifier(X, y):
# Split the data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,
random_state=42)
# Create a CountVectorizer
vectorizer = CountVectorizer()
X_train_vectorized = vectorizer.fit_transform(X_train)
X_test_vectorized = vectorizer.transform(X_test)
# Train a Naive Bayes classifier
classifier = MultinomialNB()
classifier.fit(X_train_vectorized, y_train)
# Make predictions
y_pred = classifier.predict(X_test_vectorized)
# Print classification report
print(classification_report(y_test, y_pred))
return vectorizer, classifier
def classify_text(text, vectorizer, classifier):
text_vectorized = vectorizer.transform([text])
prediction = classifier.predict(text_vectorized)
return prediction[0]
# Example usage
X=[
"I love this movie, it's amazing!",
"This book is terrible, I couldn't finish it.",
"The food at this restaurant is delicious.",
"The service here is awful, I'm never coming back.",
"What a great experience, highly recommended!",
y = ["positive", "negative", "positive", "negative", "positive"]
vectorizer, classifier = train_text_classifier(X, y)
new_text = "The product exceeded my expectations, I'm very satisfied."
prediction = classify_text(new_text, vectorizer, classifier)
print(f"Prediction for '{new_text}': {prediction}")
Building a sentiment analysis system
import nltk
from nltk.sentiment import SentimentIntensityAnalyzer
import pandas as pd
nltk.download('vader_lexicon')
def analyze_sentiment(text):
sia = SentimentIntensityAnalyzer()
sentiment_scores = sia.polarity_scores(text)
if sentiment_scores['compound'] >= 0.05:
sentiment = "Positive"
elifsentiment_scores['compound'] <= -0.05:
sentiment = "Negative"
else:
sentiment = "Neutral"
return sentiment, sentiment_scores
def analyze_sentiments(texts):
results = []
for text in texts:
sentiment, scores = analyze_sentiment(text)
results.append({
'text': text,
'sentiment': sentiment,
'pos_score': scores['pos'],
'neg_score': scores['neg'],
'neu_score': scores['neu'],
'compound_score': scores['compound']
})
return pd.DataFrame(results)
# Example usage
texts = [
"I absolutely love this product! It's amazing!",
"This is the worst experience I've ever had.",
"The movie was okay, nothing special.",
"I'm feeling pretty neutral about the whole situation.",
"The customer service was excellent and very helpful!"
results_df = analyze_sentiments(texts)
print(results_df)
Implementation of Text Summarization
from transformers import pipeline
def summarize_text(text, max_length=150, min_length=50):
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
summary = summarizer(text, max_length=max_length, min_length=min_length,
do_sample=False)
return summary[0]['summary_text']
# Example usage
long_text = """
Climate change is one of the most pressing issues facing our planet today. It refers
to long-term shifts in temperatures and weather patterns, mainly caused by human
activities, especially the burning of fossil fuels. These activities release greenhouse
gases into the atmosphere, trapping heat and causing the Earth's average
temperature to rise. The consequences of climate change are far-reaching and
include more frequent and severe weather events, rising sea levels, and disruptions
to ecosystems. To address this global challenge, countries and organizations
worldwide are working on strategies to reduce greenhouse gas emissions and
transition to cleaner energy sources. Individual actions, such as reducing energy
consumption and adopting sustainable practices, also play a crucial role in
mitigating the effects of climate change.
"""
summary = summarize_text(long_text)
print("Original text length:", len(long_text))
print("Summary length:", len(summary))
print("\nSummary:")
print(summary)
