# we will use Counter built-in object in our function
from collections import Counter 

# count words in a column of dataframe and return dataframe
def count_words(df_name, column_name, preprocess=None, min_freq=2):

    # define inner function to process tokens and update counter
    def inner_update(doc):
        tokens = doc if preprocess is None else preprocess(doc)
        counter.update(tokens) # update here is counter method

    # create counter and run through all data
    counter = Counter()
    df_name[column_name].map(inner_update) # call inner function

    # transform counter into data frame
    freq_df = pd.DataFrame.from_dict(counter, orient='index', columns=['freq'])
    # query the columns of a DataFrame with a boolean expression
    freq_df = freq_df.query('freq >= @min_freq')
    # change the index name
    freq_df.index.name = 'token'
    
    return freq_df.sort_values('freq', ascending=False)

import pandas as pd
import spacy
# load a pipeline package by name and return nlp object
nlp = spacy.load("en_core_web_sm")

details = {
    'Text_id' : [23, 21, 22, 21],
    'Text' : ['All roads lead to Rome', 
              'All work and no play makes Jack a dull buy', 
              'Any port in a storm', 
              'Avoid a questioner, for he is also a tattler'],
}
  
# creating a Dataframe object 
example_df = pd.DataFrame(details)

# count the words after tokenization (stop words and punctuations are EXCLUDED)
# use spaCys nlp object for the preprocessing

count_words(example_df,
            "Text",
           preprocess = lambda x: [t.text for t in nlp(x) if not t.is_stop and not t.is_punct],
           min_freq=1)

            freq
token           
roads          1
lead           1
Rome           1
work           1
play           1
makes          1
Jack           1
dull           1
buy            1
port           1
storm          1
Avoid          1
questioner     1
tattler        1