Calculate number of tokens

def calculate_number_of_tokens(string):
    tokens = string.split()
    return len(tokens)

string = "This is a sample string with some tokens"

num_tokens = calculate_number_of_tokens(string)

print("Number of tokens:", num_tokens)

Leave a Reply

Your email address will not be published. Required fields are marked *