Natural Language Processing with Disaster Tweets End to End Project

# Import necessary libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import classification_report, accuracy_score, confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression

# Load the datasets
train_data = pd.read_csv('/path/to/train.csv')  # Replace with actual file path
test_data = pd.read_csv('/path/to/test.csv')

# Preview the datasets
train_data.head()

# Display dataset information
train_data.info()
train_data.describe()
train_data.isnull().sum()

# Plot the distribution of the target variable
sns.countplot(x='target', data=train_data, palette='viridis')
plt.title('Distribution of Target Variable')
plt.xlabel('Target (0 = Not Disaster, 1 = Disaster)')
plt.ylabel('Count')
plt.show()

# Most frequent keywords
top_keywords = train_data['keyword'].value_counts().head(10)
top_keywords.plot(kind='bar', color='blue')
plt.title('Top 10 Keywords')
plt.xlabel('Keyword')
plt.ylabel('Count')
plt.show()

# Length of tweets
train_data['tweet_length'] = train_data['text'].apply(len)
sns.histplot(train_data['tweet_length'], kde=True, color='green')
plt.title('Tweet Length Distribution')
plt.xlabel('Tweet Length')
plt.ylabel('Frequency')
plt.show()

# Fill missing keywords and locations
train_data['keyword'] = train_data['keyword'].fillna('no_keyword')
train_data['location'] = train_data['location'].fillna('unknown')

# Drop rows with missing text
data = train_data.dropna(subset=['text'])

# Import NLTK packages
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
import re

# Download necessary NLTK resources
nltk.download('stopwords')
nltk.download('punkt')
nltk.download('wordnet')

# Text preprocessing function
def clean_text(text):
    text = re.sub(r'http\S+', '', text)  # Remove URLs
    text = re.sub(r'[^a-zA-Z\s]', '', text)  # Remove special characters
    text = text.lower()  # Convert to lowercase
    tokens = word_tokenize(text)  # Tokenize
    tokens = [word for word in tokens if word not in stopwords.words('english')]  # Remove stopwords
    lemmatizer = WordNetLemmatizer()
    tokens = [lemmatizer.lemmatize(word) for word in tokens]  # Lemmatize
    return ' '.join(tokens)

# Apply text cleaning
train_data['clean_text'] = train_data['text'].apply(clean_text)

# Initialize TF-IDF Vectorizer
tfidf = TfidfVectorizer(max_features=5000)

# Transform text data
X = tfidf.fit_transform(train_data['clean_text']).toarray()
y = train_data['target']

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Train logistic regression model
logreg = LogisticRegression()
logreg.fit(X_train, y_train)

# Make predictions
y_pred_logreg = logreg.predict(X_test)

# Evaluate the model
print("Logistic Regression Accuracy:", accuracy_score(y_test, y_pred_logreg))
print(classification_report(y_test, y_pred_logreg))

# Train Random Forest model
rf_model = RandomForestClassifier()
rf_model.fit(X_train, y_train)

# Make predictions
y_pred_rf = rf_model.predict(X_test)

# Evaluate the model
print("Random Forest Accuracy:", accuracy_score(y_test, y_pred_rf))
print(classification_report(y_test, y_pred_rf))

# Preprocess test data
test_data['clean_text'] = test_data['text'].apply(clean_text)
X_test_final = tfidf.transform(test_data['clean_text']).toarray()

# Predict using the best model
test_data['target'] = rf_model.predict(X_test_final)

# Save the submission file
test_data[['id', 'target']].to_csv('submission.csv', index=False)