🗣️ Computational Linguistics & NLP

import nltk import re from nltk.corpus import stopwords from nltk.tokenize import word_tokenize, sent_tokenize from nltk.stem import WordNetLemmatizer from nltk.chunk import ne_chunk from nltk.tag import pos_tag class TextPreprocessor: def __init__(self): # Download required NLTK data nltk.download(['punkt', 'stopwords', 'wordnet', 'averaged_perceptron_tagger', 'maxent_ne_chunker', 'words']) self.stop_words = set(stopwords.words('english')) self.lemmatizer = WordNetLemmatizer() def clean_text(self, text): """Clean and normalize text""" # Remove HTML tags text = re.sub(r'<[^>]+>', '', text) # Remove special characters and digits text = re.sub(r'[^a-zA-Z\s]', '', text) # Convert to lowercase text = text.lower() # Remove extra whitespace text = re.sub(r'\s+', ' ', text).strip() return text def tokenize_and_preprocess(self, text): """Complete preprocessing pipeline""" # Clean text clean_text = self.clean_text(text) # Tokenize tokens = word_tokenize(clean_text) # Remove stop words and lemmatize processed_tokens = [ self.lemmatizer.lemmatize(token) for token in tokens if token not in self.stop_words and len(token) > 2 ] return processed_tokens def extract_entities(self, text): """Extract named entities""" tokens = word_tokenize(text) pos_tags = pos_tag(tokens) entities = ne_chunk(pos_tags) named_entities = [] for chunk in entities: if hasattr(chunk, 'label'): entity = ' '.join([token for token, pos in chunk.leaves()]) named_entities.append((entity, chunk.label())) return named_entities # Usage example preprocessor = TextPreprocessor() sample_text = """ Natural Language Processing (NLP) is a branch of artificial intelligence that helps computers understand, interpret and manipulate human language. Apple Inc. and Google are leading companies in this field. """ # Preprocess text processed_tokens = preprocessor.tokenize_and_preprocess(sample_text) print("Processed tokens:", processed_tokens) # Extract named entities entities = preprocessor.extract_entities(sample_text) print("Named entities:", entities)

from transformers import AutoTokenizer, AutoModelForSequenceClassification from transformers import pipeline import torch import numpy as np class SentimentAnalyzer: def __init__(self, model_name="cardiffnlp/twitter-roberta-base-sentiment-latest"): """Initialize sentiment analysis model""" self.tokenizer = AutoTokenizer.from_pretrained(model_name) self.model = AutoModelForSequenceClassification.from_pretrained(model_name) # Create pipeline for easy inference self.sentiment_pipeline = pipeline( "sentiment-analysis", model=self.model, tokenizer=self.tokenizer, device=0 if torch.cuda.is_available() else -1 ) def analyze_sentiment(self, text): """Analyze sentiment of a single text""" result = self.sentiment_pipeline(text) return { 'label': result[0]['label'], 'confidence': result[0]['score'] } def batch_analyze(self, texts): """Analyze sentiment for multiple texts""" results = self.sentiment_pipeline(texts) return [ { 'text': text, 'label': result['label'], 'confidence': result['score'] } for text, result in zip(texts, results) ] def detailed_analysis(self, text): """Get detailed sentiment scores""" inputs = self.tokenizer(text, return_tensors="pt", truncation=True, padding=True) with torch.no_grad(): outputs = self.model(**inputs) predictions = torch.nn.functional.softmax(outputs.logits, dim=-1) labels = ['NEGATIVE', 'NEUTRAL', 'POSITIVE'] scores = predictions.cpu().numpy()[0] return {label: float(score) for label, score in zip(labels, scores)} # Usage example analyzer = SentimentAnalyzer() # Sample texts for analysis sample_texts = [ "I love this new AI technology! It's amazing.", "The weather is okay today, nothing special.", "I hate when my computer crashes during important work." ] # Analyze individual text result = analyzer.analyze_sentiment(sample_texts[0]) print(f"Sentiment: {result['label']} (confidence: {result['confidence']:.3f})") # Batch analysis batch_results = analyzer.batch_analyze(sample_texts) for result in batch_results: print(f"Text: {result['text']}") print(f"Sentiment: {result['label']} ({result['confidence']:.3f})\n") # Detailed analysis detailed = analyzer.detailed_analysis(sample_texts[0]) print("Detailed sentiment scores:", detailed)