import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.display import Markdown, display
import plotly.io as pio
pio.renderers.default = "iframe_connected"  # allows for stable ploty rendering for HTML export

# Set pandas to display the full content of columns
pd.set_option('display.max_colwidth', None) 

# Read-in data into our dataframe
df = pd.read_csv("/Users/rileysvensson/Desktop/CAA - Data Analyst/Projects/imdb_data/final/IMDB_clean/IMDB.csv")

# Clean the 'Genres' column by removing "Back to top" text from our scraping
df["Genres"] = df["Genres"].str.replace("Back to top", "", regex=False).str.strip()


# Define the set of well-known genres to extract from niche genres
standard_genres = {
    "Action", "Adult", "Adventure", "Animation", "Biography", "Comedy", "Crime", "Documentary",
    "Drama", "Family", "Fantasy", "Film Noir", "Game Show", "History", "Horror", "Musical",
    "Music", "Mystery", "News", "Reality-TV", "Romance", "Sci-Fi", "Short", "Sport",
    "Talk-Show", "Thriller", "War", "Western"
}

# Function to strictly extract standard genres if they appear anywhere in the genre string
def extract_standard_genres(genre_str):
    if pd.isna(genre_str):
        return None  # Handle missing values
    genre_str_lower = genre_str.lower()  # Convert to lowercase for matching
    filtered_genres = {std_genre for std_genre in standard_genres if std_genre.lower() in genre_str_lower}
    return ", ".join(filtered_genres) if filtered_genres else None

# Apply the strict genre extraction function
df["Cleaned_Genres"] = df["Genres"].apply(extract_standard_genres)

# Create a binary column for "Highly Rated" movies (IMDb rating >= 7.0)
df["Highly_Rated"] = (df["Rating"] >= 7).astype(int)

# Function to clean and deduplicate actor names within each row, handling NAN's
def clean_stars_column(stars):
    if pd.isna(stars):
        return ""  
    
    # Split name by commas, strip spaces, standardize formatting, and remove duplicates
    unique_names = sorted(set(name.strip() for name in stars.split(",")))
    
    # Rejoin into a single, clean string
    return ", ".join(unique_names)

# Apply function to "Stars" column
df["Stars"] = df["Stars"].apply(clean_stars_column)

# Create function to count # of words in the movie description
def count_words(description):
    
    # Check if "Read all", "..." or "--" is in the text and remove truncated parts
    truncated = False
    for marker in ["Read all", "...", "--"]:
        if marker in description:
            description = description.split(marker)[0].strip()
            truncated = True

    # Count words in the cleaned description
    word_count = len(description.split())

    # Indicate if the description was truncated, by adding +
    return word_count if not truncated else f"{word_count}+"  

# Apply function to Description column
df["Description Word Count"] = df["Description"].apply(count_words)

# Create function to bin word count
def categorize_word_count(word_count):
    count = int(str(word_count).replace("+", ""))
    return "<5" if count < 5 else "5-10" if count < 10 else "10-15" if count < 15 else "15-20" if count < 20 else "20-25" if count < 25 else "25-30" if count < 30 else "30+"

# Apply binning function
df["Word_Count_Binned"] = df["Description Word Count"].apply(categorize_word_count)

# Count number of unique genres
df["Num_Genres"] = df["Cleaned_Genres"].apply(lambda x: len(x.split(",")) if pd.notna(x) and x.strip() else 0)

# Keep only rows where 'Year' starts with 19 or 20
df = df[df["Year"].astype(str).str.match(r"^(19|20)\d{2}$")]

# Convert Year to numeric
df["Year"] = pd.to_numeric(df["Year"])

# Function to extract the first genre from "Cleaned_Genres"
def extract_primary_genre(genre_str):
    if pd.isna(genre_str) or genre_str.strip() == "":
        return None  # Handle missing values
    return genre_str.split(",")[0].strip()  # Take the first genre

# Apply function to create the "Cleaned_Primary_Genre" column
df["Cleaned_Primary_Genre"] = df["Cleaned_Genres"].apply(extract_primary_genre)

# Create function to create new column 1 if the Writer of a movie is also the Director, 0 otherwise
def director_is_writer(row):
    director = str(row['Director']).strip().lower()
    writers = str(row['Writers']).split(',')
    writers_cleaned = [w.strip().lower() for w in writers]
    return int(director in writers_cleaned)

df['Director_Is_Writer'] = df.apply(director_is_writer, axis=1)

# Drop movies with a duration greater than 300 minutes, as these were very obscure outliers
df = df[df["Duration"] <= 300]

# Display Forest Gump as an example record
display_movie_ex = df[df['Title'].isin(['Forrest Gump',''])]

display_movie_ex

import pandas as pd
from IPython.display import HTML

# Define glossary as a list of dictionaries
glossary_data = [
    {"Variable": "Title", "Description": "The name of the movie"},
    {"Variable": "Year", "Description": "The year the movie was released"},
    {"Variable": "Genres", "Description": "All genres from IMDb"},
    {"Variable": "Cleaned_Genres", "Description": "Filtered version of `Genres` with only standard genres"},
    {"Variable": "Cleaned_Primary_Genre", "Description": "First genre listed in `Cleaned_Genres`"},
    {"Variable": "Rating", "Description": "IMDb rating out of 10"},
    {"Variable": "Votes", "Description": "Total number of IMDb votes received (as of March 2025)"},
    {"Variable": "Duration", "Description": "Runtime in minutes"},
    {"Variable": "Certificate", "Description": "Age suitability rating (e.g., PG, PG-13, R)"},
    {"Variable": "Stars", "Description": "List of lead actors"},
    {"Variable": "Writers", "Description": "List of lead writers"},
    {"Variable": "Director", "Description": "Main director"},
    {"Variable": "Description", "Description": "IMDb plot summary"},
    {"Variable": "Description Word Count", "Description": "Word count of the description"},
    {"Variable": "Word_Count_Binned", "Description": "Binned version of description length (e.g., <5, 10-15, 30+)"},
    {"Variable": "Num_Genres", "Description": "Count of genres per movie"},
    {"Variable": "Highly_Rated", "Description": "1 if movie’s rating is above 7.0, otherwise 0"},
    {"Variable": "Director_Is_Writer", "Description": "1 if the director is also a writer (creative control)"},
]


# Convert DataFrame to HTML without index
glossary_html = glossary_df.to_html(index=False, escape=False)

# Display
display(HTML(glossary_html))

import plotly.graph_objects as go
import pandas as pd
import plotly.io as pio

import plotly.io as pio
pio.renderers.default = "plotly_mimetype+notebook"

# Filter out invalid years (keeping only reasonable movie years)
df_movie_count_over_time = df[(df['Year'] >= 1940) & (df['Year'] <= 2023)]

# Count movies per year
movie_counts = df_movie_count_over_time["Year"].value_counts().sort_index()
df_movies = pd.DataFrame({'Year': movie_counts.index, 'Count': movie_counts.values})

fig = go.Figure(go.Bar(
    x=df_movies['Year'], 
    y=df_movies['Count'], 
    marker_color="#E6B91E"
))

fig.update_layout(
    plot_bgcolor="black",
    paper_bgcolor="black",
    title=dict(text="🎬 Movies by Year", font=dict(size=26, color="gold"), x=0.5),
    xaxis=dict(
        title="Year",
        tickmode='linear',
        dtick=5,
        tickfont=dict(size=14, color='white'),
        title_font=dict(size=16, color='white'),
        linecolor='white'
    ),
    yaxis=dict(
        title="Number of Movies",
        gridcolor="white",
        tickfont=dict(size=14, color='white'),
        title_font=dict(size=16, color='white'),
        linecolor='white'
    ),
    showlegend=False
)

fig

import plotly.graph_objects as go
import pandas as pd
import plotly.io as pio

# Use this to render inside Quarto or Jupyter Notebook
pio.renderers.default = "plotly_mimetype+notebook"

# Filter out invalid years
df_rating_over_time = df[(df['Year'] >= 1940) & (df['Year'] <= 2023)]

# Calculate average rating per year
avg_rating_per_year = df_rating_over_time.groupby("Year")["Rating"].mean().reset_index()

# Create line chart
fig = go.Figure()

fig.add_trace(go.Scatter(
    x=avg_rating_per_year["Year"],
    y=avg_rating_per_year["Rating"],
    mode="lines+markers",
    marker=dict(color="red", size=6),
    line=dict(color="red", width=2),
    name="Average Rating"
))

# Update layout to match your bar chart theme
fig.update_layout(
    plot_bgcolor="black",
    paper_bgcolor="black",
    title=dict(text="⭐ Average IMDb Rating Over Time", font=dict(size=26, color="gold"), x=0.5),
    xaxis=dict(
        title="Year",
        tickmode='linear',
        dtick=5,
        tickfont=dict(size=14, color='white'),
        title_font=dict(size=16, color='white'),
        linecolor='white',
        gridcolor='white'
    ),
    yaxis=dict(
        title="Average Rating",
        range=[5, 9],
        tickfont=dict(size=14, color='white'),
        title_font=dict(size=16, color='white'),
        linecolor='white',
        gridcolor='white'
    ),
    showlegend=False
)

fig

from IPython.display import HTML

display(HTML('''
<table style="width:100%; border-collapse: collapse; margin: 2em 0; font-size:15px;">
  <thead style="background-color:#f9f9f9;">
    <tr>
      <th style="padding:10px; border: 1px solid #999;">Metric</th>
      <th style="padding:10px; border: 1px solid #999;">What It Means</th>
      <th style="padding:10px; border: 1px solid #999;">Why It Matters</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td style="padding:10px; border: 1px solid #999;"><strong>RMSE</strong> (Root Mean Squared Error)</td>
      <td style="padding:10px; border: 1px solid #999;">Shows how far off the model’s predictions are from the actual movie ratings, on average.</td>
      <td style="padding:10px; border: 1px solid #999;">An RMSE of 0.89 means the model is usually within 1 star of the actual IMDb rating (on a 10-star scale).</td>
    </tr>
    <tr>
      <td style="padding:10px; border: 1px solid #999;"><strong>R² Score</strong> (Coefficient of Determination)</td>
      <td style="padding:10px; border: 1px solid #999;">Tells us how well the model explains the differences in movie ratings.</td>
      <td style="padding:10px; border: 1px solid #999;">A higher R² means the model does a better job at understanding what drives audience scores.</td>
    </tr>
  </tbody>
</table>
'''))

import os
import pandas as pd
import numpy as np
import tensorflow as tf
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler, MinMaxScaler
from tensorflow.keras import layers, models
from tensorflow.keras.regularizers import l2
from sklearn.metrics import mean_squared_error, r2_score
from IPython.display import Markdown, display

# Suppress TensorFlow logs
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# ========================
# 1. Feature Configuration
# ========================

categorical_features = ["Certificate", "Word_Count_Binned","Director", "Cleaned_Primary_Genre", "Director_Is_Writer"]
numerical_features = ["Duration", "Num_Genres", "Year"]

drop_cols = ["IMDB_ID", "Title", "Highly_Rated", "Stars","Writers",
             "Genres", "Description", "Primary Genre", 
             "Description Word Count", "Votes"]

# ========================
# 2. Data Preparation
# ========================

df_movies_features = df.drop(columns=drop_cols, errors="ignore").dropna()

X = df_movies_features[categorical_features + numerical_features]
y = df_movies_features["Rating"].values.reshape(-1, 1)

# Normalize target
scaler_y = MinMaxScaler()
y = scaler_y.fit_transform(y)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# ========================
# 3. Preprocessing Pipeline
# ========================

ct_all = ColumnTransformer([
    ("cat", Pipeline([
        ("imputer", SimpleImputer(strategy="constant", fill_value="Unknown")),
        ("onehot", OneHotEncoder(sparse_output=False, handle_unknown="ignore"))
    ]), categorical_features),

    ("num", Pipeline([
        ("imputer", SimpleImputer(strategy="median")),
        ("scaler", StandardScaler())
    ]), numerical_features)
]).set_output(transform="pandas")

X_train = ct_all.fit_transform(X_train)
X_test = ct_all.transform(X_test)

X_train = np.array(X_train, dtype=np.float32)
X_test = np.array(X_test, dtype=np.float32)

# ========================
# 4. Neural Network Training
# ========================

model = models.Sequential([
    layers.Input(shape=(X_train.shape[1],)),
    layers.Dense(256, activation="relu", kernel_regularizer=l2(0.01)),
    layers.Dropout(0.3),
    layers.Dense(128, activation="relu", kernel_regularizer=l2(0.01)),
    layers.Dropout(0.3),
    layers.Dense(64, activation="relu"),
    layers.Dense(1)
])

model.compile(optimizer="adam", loss="mse", metrics=["mae"])

history = model.fit(X_train, y_train, epochs=200, batch_size=16, validation_data=(X_test, y_test), verbose=0)

# ========================
# 5. Evaluate Model
# ========================

test_loss, test_mae = model.evaluate(X_test, y_test, verbose=1)

y_pred = model.predict(X_test)
y_pred = scaler_y.inverse_transform(y_pred)
y_test = scaler_y.inverse_transform(y_test)

nn_rmse = np.sqrt(mean_squared_error(y_test, y_pred))
nn_r2 = r2_score(y_test, y_pred)

display(Markdown(f"""
### 🧠 Neural Network Performance
**Features used:** {', '.join(categorical_features + numerical_features)}  
**Neural Network RMSE:** {nn_rmse:.2f}  
**Neural Network R² Score:** {nn_r2:.2f}
"""))

import os
import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split, cross_val_score, KFold
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler, MinMaxScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_squared_error, r2_score
from IPython.display import Markdown, display

# Suppress TensorFlow logs (in case you're switching between models)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# ========================
# 1. Feature Configuration
# ========================

categorical_features = ["Certificate", "Word_Count_Binned", "Director", "Cleaned_Primary_Genre", "Director_Is_Writer"]
numerical_features = ["Duration", "Num_Genres", "Year"]

drop_cols = ["IMDB_ID", "Title", "Highly_Rated", "Stars", "Writers",
             "Genres", "Description", "Primary Genre",
             "Description Word Count", "Votes"]

# ========================
# 2. Data Preparation
# ========================

df_movies_features = df.drop(columns=drop_cols, errors="ignore").dropna()

X = df_movies_features[categorical_features + numerical_features]
y = df_movies_features["Rating"].values.reshape(-1, 1)

scaler_y = MinMaxScaler()
y_scaled = scaler_y.fit_transform(y)

X_train, X_test, y_train, y_test = train_test_split(X, y_scaled, test_size=0.3, random_state=43)

# ========================
# 3. Preprocessing Pipeline
# ========================

ct_all = ColumnTransformer([
    ("cat", Pipeline([
        ("imputer", SimpleImputer(strategy="constant", fill_value="Unknown")),
        ("onehot", OneHotEncoder(sparse_output=False, handle_unknown="ignore"))
    ]), categorical_features),

    ("num", Pipeline([
        ("imputer", SimpleImputer(strategy="median")),
        ("scaler", StandardScaler())
    ]), numerical_features)
]).set_output(transform="pandas")

X_train = ct_all.fit_transform(X_train)
X_test = ct_all.transform(X_test)

X_train = np.array(X_train, dtype=np.float32)
X_test = np.array(X_test, dtype=np.float32)

# ========================
# 4. Train Random Forest
# ========================

rf_model = RandomForestRegressor(n_estimators=200, random_state=2, n_jobs=-1)
rf_model.fit(X_train, y_train.ravel())

y_pred_rf = rf_model.predict(X_test)

y_pred_rf = scaler_y.inverse_transform(y_pred_rf.reshape(-1, 1))
y_test_original = scaler_y.inverse_transform(y_test)

# ========================
# 5. Evaluate Model (Train/Test Split)
# ========================

rf_rmse = np.sqrt(mean_squared_error(y_test_original, y_pred_rf))
rf_r2 = r2_score(y_test_original, y_pred_rf)

display(Markdown(f"""
### 🌳 Random Forest Performance
**Features used:** {', '.join(categorical_features + numerical_features)}  
**Random Forest RMSE:** {rf_rmse:.2f}  
**Random Forest R² Score:** {rf_r2:.2f}
"""))

# ========================
# 6. Cross-Validation (Optional but Robust)
# ========================

# Use original unscaled y for meaningful R²
y_unscaled = y.ravel()

# Create full pipeline using the existing ColumnTransformer
pipeline_rf = make_pipeline(
    ct_all,
    RandomForestRegressor(n_estimators=200, random_state=2, n_jobs=-1)
)

# Set up cross-validation
kf = KFold(n_splits=5, shuffle=True, random_state=42)
cv_r2_scores = cross_val_score(pipeline_rf, X, y_unscaled, cv=kf, scoring="r2")

# Display results
display(Markdown(f"""
#### Cross-Validation Performance (5-Fold)
**Average R² Score:** {cv_r2_scores.mean():.3f}  
**R² Scores per Fold:** {', '.join(f'{score:.3f}' for score in cv_r2_scores)}
"""))

import os
import pandas as pd
import numpy as np
from xgboost import XGBRegressor
from sklearn.impute import SimpleImputer
from sklearn.model_selection import train_test_split
from sklearn.compose import ColumnTransformer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder, StandardScaler, MinMaxScaler
from sklearn.metrics import mean_squared_error, r2_score

# Suppress TensorFlow logs
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# ========================
# 1. Feature Configuration
# ========================

categorical_features = ["Certificate", "Word_Count_Binned", "Director", "Cleaned_Primary_Genre", "Director_Is_Writer"]
numerical_features = ["Duration", "Num_Genres", "Year"]

drop_cols = ["IMDB_ID", "Title", "Highly_Rated", "Stars", "Writers",
             "Genres", "Description", "Primary Genre",
             "Description Word Count", "Votes"]

# ========================
# 2. Data Preparation
# ========================

df_movies_features = df.drop(columns=drop_cols, errors="ignore").dropna()

X = df_movies_features[categorical_features + numerical_features]
y = df_movies_features["Rating"].values.reshape(-1, 1)

scaler_y = MinMaxScaler()
y = scaler_y.fit_transform(y)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# ========================
# 3. Preprocessing Pipeline
# ========================

ct_all = ColumnTransformer([
    ("cat", Pipeline([
        ("imputer", SimpleImputer(strategy="constant", fill_value="Unknown")),
        ("onehot", OneHotEncoder(sparse_output=False, handle_unknown="ignore"))
    ]), categorical_features),

    ("num", Pipeline([
        ("imputer", SimpleImputer(strategy="median")),
        ("scaler", StandardScaler())
    ]), numerical_features)
]).set_output(transform="pandas")

X_train = ct_all.fit_transform(X_train)
X_test = ct_all.transform(X_test)

X_train = np.array(X_train, dtype=np.float32)
X_test = np.array(X_test, dtype=np.float32)

# ========================
# 4. Train XGBoost Regressor
# ========================

xgb_model = XGBRegressor(n_estimators=200, max_depth=6, learning_rate=0.1, random_state=42, n_jobs=-1)
xgb_model.fit(X_train, y_train.ravel())

# Predict
y_pred_xgb = xgb_model.predict(X_test)

# Rescale predictions
y_pred_xgb = scaler_y.inverse_transform(y_pred_xgb.reshape(-1, 1))
y_test_original = scaler_y.inverse_transform(y_test)

# ========================
# 5. Evaluate Model
# ========================

xgb_rmse = np.sqrt(mean_squared_error(y_test_original, y_pred_xgb))
xgb_r2 = r2_score(y_test_original, y_pred_xgb)

display(Markdown(f"""
### 🔋 Extreme Gradient (XG) Boost Performance
**Features used:** {', '.join(categorical_features + numerical_features)}  
**XGB RMSE:** {xgb_rmse:.2f}  
**XGB R² Score:** {xgb_r2:.2f}
"""))

from IPython.display import HTML

display(HTML('''
<h3 style="text-align:center; margin-top:30px;">Model Showdown: A Tight Cut</h3>
<table style="width:100%; border-collapse: collapse; font-size:15px; margin: 2em 0;">
  <thead style="background-color:#f9f9f9;">
    <tr>
      <th style="padding:10px; border: 1px solid #999;">Model Type</th>
      <th style="padding:10px; border: 1px solid #999;">Strengths</th>
      <th style="padding:10px; border: 1px solid #999;">Weaknesses</th>
      <th style="padding:10px; border: 1px solid #999;">RMSE</th>
      <th style="padding:10px; border: 1px solid #999;">R² Score</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td style="padding:10px; border: 1px solid #999;">Random Forest</td>
      <td style="padding:10px; border: 1px solid #999;">Easy to interpret, handles mixed data well</td>
      <td style="padding:10px; border: 1px solid #999;">Slightly less flexible than XGBoost</td>
      <td style="padding:10px; border: 1px solid #999;">0.87</td>
      <td style="padding:10px; border: 1px solid #999;">0.289</td>
    </tr>
    <tr>
      <td style="padding:10px; border: 1px solid #999;">XGBoost</td>
      <td style="padding:10px; border: 1px solid #999;">Captures complex nonlinear interactions</td>
      <td style="padding:10px; border: 1px solid #999;">More complex to interpret</td>
      <td style="padding:10px; border: 1px solid #999;">0.89</td>
      <td style="padding:10px; border: 1px solid #999;">0.28</td>
    </tr>
  </tbody>
</table>
'''))

import pandas as pd
import re
from rapidfuzz import process, fuzz
from collections import defaultdict

# Step 1: Clean title
def clean_title(title):
    title = title.lower()
    title = re.sub(r'[^\w\s]', '', title)
    return re.sub(r'\s+', ' ', title).strip()

df_titles = df[["Title", "Rating", "Year"]].copy()
df_titles["clean_title"] = df_titles["Title"].apply(clean_title)

# Step 2: Extract first strong word (excluding filler words)
filler_words = {"the", "a", "an", "of", "and", "in", "on", "at", "to", "for"}
def get_main_word(title):
    words = title.split()
    for word in words:
        if word not in filler_words:
            return word
    return words[0] if words else "unknown"

df_titles["block_key"] = df_titles["clean_title"].apply(get_main_word)

# Step 3: Group by block_key
df_titles["cluster_id"] = -1
matched = set()
clusters = defaultdict(list)
cluster_id = 0

for key, group in df_titles.groupby("block_key"):
    if len(group) == 1:
        idx = group.index[0]
        df_titles.at[idx, "cluster_id"] = cluster_id
        cluster_id += 1
        continue  # skip singletons
    
    titles = group["clean_title"].tolist()
    index_map = dict(zip(titles, group.index))
    
    for i, title in enumerate(titles):
        if index_map[title] in matched:
            continue
        results = process.extract(title, titles, scorer=fuzz.token_set_ratio, score_cutoff=70)
        matched_idxs = [index_map[r[0]] for r in results if index_map[r[0]] not in matched]
        
        for idx in matched_idxs:
            matched.add(idx)
            clusters[cluster_id].append(idx)
        cluster_id += 1

# Step 4: Assign back
for cid, indices in clusters.items():
    df_titles.loc[indices, "cluster_id"] = cid

df_titles = df_titles.sort_values(["cluster_id", "Year"])

# Filter to only clusters with more than 1 member
non_singleton_clusters = df_titles["cluster_id"].value_counts()
non_singleton_clusters = non_singleton_clusters[non_singleton_clusters > 1].index

df_clustered = df_titles[
    df_titles["cluster_id"].isin(non_singleton_clusters) & (df_titles["cluster_id"] != -1)
].sort_values(["cluster_id", "Year"])

import pandas as pd
import plotly.graph_objects as go
from scipy.stats import ttest_ind

# Load and clean
df_series_remakes = pd.read_csv("fuzzy_matching_clusters_sanity_check_2.csv")
df_series_remakes = df_series_remakes[df_series_remakes['cluster_id'] != -1]
df_series_remakes = df_series_remakes.sort_values(by=['cluster_id', 'Year'])

# Tag original vs. follower
def tag_role(group):
    group = group.sort_values(by='Year')
    group['role'] = ['Originals'] + ['Followers'] * (len(group) - 1)
    return group

df_tagged = df_series_remakes.groupby('cluster_id', group_keys=False).apply(tag_role).reset_index(drop=True)

# Extract ratings
originals = df_tagged[df_tagged['role'] == 'Originals']['Rating']
followers = df_tagged[df_tagged['role'] == 'Followers']['Rating']

# T-test
t_stat, p_val = ttest_ind(originals, followers, equal_var=False)

# Average ratings
avg_ratings = df_tagged.groupby('role')['Rating'].mean().reindex(['Originals', 'Followers'])

# IMDb yellow & Netflix red
bar_colors = ['#F5C518', '#E50914']

# Plot
fig = go.Figure()

fig.add_trace(go.Bar(
    x=avg_ratings.index,
    y=avg_ratings.values,
    text=[f"{val:.2f}" for val in avg_ratings.values],
    textposition='outside',
    marker_color=bar_colors,
    hoverinfo='x+y',
))

# Annotate t-test result (moved lower + left-aligned)
fig.add_annotation(
    x=0.0,
    y=1.0,
    text=f"T-stat: {t_stat:.2f} | P-value: {p_val:.4f}",
    showarrow=False,
    font=dict(size=12, color="white"),
    align='center',
    bgcolor="#333333",
    bordercolor="#888",
    borderwidth=1
)

# Layout
fig.update_layout(
    title=dict(
        text='Average IMDb Rating: Originals vs. Followers 🍿',
        x=0.5,
        xanchor='center',
        font=dict(color='#F5C518', size=22)
    ),
    yaxis=dict(title='Average Rating', range=[0, 10]),
    xaxis=dict(title='Film Type'),
    plot_bgcolor='#1C1C1C',
    paper_bgcolor='#1C1C1C',
    font=dict(family="Arial", size=14, color="#FFFFFF")
)

fig.show()

import pandas as pd
from IPython.display import Markdown, display

# Start fresh with a copy of your existing DataFrame
df_clustered = df_series_remakes.copy()

# Filter out unclustered (-1)
df_clustered = df_clustered[df_clustered['cluster_id'] != -1]

# Sort by cluster and year
df_clustered = df_clustered.sort_values(by=['cluster_id', 'Year'])

# Label roles: first = original, rest = follower
def assign_roles(group):
    group = group.sort_values(by='Year')
    group['role_new'] = ['original'] + ['follower'] * (len(group) - 1)
    return group

df_clustered_labeled = df_clustered.groupby('cluster_id', group_keys=False).apply(assign_roles)

# Compare followers to original
comparison_rows_new = []

for cluster_id, group in df_clustered_labeled.groupby('cluster_id'):
    group = group.sort_values(by='Year')
    if len(group) < 2:
        continue
    original = group.iloc[0]
    for _, follower in group.iloc[1:].iterrows():
        comparison_rows_new.append({
            'cluster_id': cluster_id,
            'original_title': original['Title'],
            'original_rating': original['Rating'],
            'follower_title': follower['Title'],
            'follower_rating': follower['Rating'],
            'follower_same_or_better': follower['Rating'] >= original['Rating']
        })

# Create new comparison DataFrame
df_follower_vs_original = pd.DataFrame(comparison_rows_new)

# Summary stats
total_comparisons_new = len(df_follower_vs_original)
better_or_equal_new = df_follower_vs_original['follower_same_or_better'].sum()
percentage_new = (better_or_equal_new / total_comparisons_new) * 100

display(Markdown(f"""
 Out of **{total_comparisons_new}** remake comparisons,  
 **{better_or_equal_new}** were rated the same or better than the original.  
 That’s **{percentage_new:.2f}%** of the time.
"""))

# ========================
# 6. Predict for All Movies in File
# ========================

# Load the file (CSV or Excel)
df_predict = pd.read_excel("/Users/rileysvensson/Desktop/CAA - Data Analyst/Projects/imdb_data/final/predict_movies.xlsx")  

# Drop unneeded columns
X_new = df_predict.drop(columns=drop_cols, errors="ignore")

# Transform using the pipeline
X_new_trans = ct_all.transform(X_new[categorical_features + numerical_features])
X_new_trans = np.array(X_new_trans, dtype=np.float32)

# Predict and rescale
y_preds_scaled = rf_model.predict(X_new_trans)
y_preds_original = scaler_y.inverse_transform(y_preds_scaled.reshape(-1, 1))

# Attach predictions to original dataframe
df_predict["Predicted_Rating"] = y_preds_original

# Sort by predicted rating (highest to lowest)
df_predict_sorted = df_predict.sort_values("Predicted_Rating", ascending=False)

# Print title and predicted rating for each movie
for title, rating in zip(df_predict_sorted["Title"], df_predict_sorted["Predicted_Rating"]):
    print(f"🎬 {title}: {rating:.2f}")

import seaborn as sns
import matplotlib.pyplot as plt

from tabulate import tabulate

# Get feature names after transformation
feature_names = ct_all.get_feature_names_out()

# Get feature importances
importances = rf_model.feature_importances_

# Create a DataFrame for better readability
feature_importance_df = pd.DataFrame({
    "Feature": feature_names,
    "Importance": importances
}).sort_values(by="Importance", ascending=False)

# Convert top 20 to markdown table
top_20_features = feature_importance_df.head(20)

#
#| Feature                                |   Importance |
#|----------------------------------------|--------------|
#| num__Duration                          |   0.175685   |
#| num__Year                              |   0.117457   |
#| num__Num_Genres                        |   0.0316729  |
#| cat__Cleaned_Primary_Genre_Drama       |   0.0293566  |
#| cat__Cleaned_Primary_Genre_Documentary |   0.0179949  |
#| cat__Cleaned_Primary_Genre_Biography   |   0.0126313  |
#| cat__Cleaned_Primary_Genre_Action      |   0.0086014  |
#| cat__Word_Count_Binned_30+             |   0.00728996 |
#| cat__Certificate_Not Rated             |   0.00728064 |
#| cat__Certificate_R                     |   0.00695221 |
#| cat__Certificate_PG-13                 |   0.006874   |
#| cat__Director_Is_Writer_1              |   0.00664878 |
#| cat__Director_Is_Writer_0              |   0.00639114 |
#| cat__Word_Count_Binned_20-25           |   0.0063814  |
#| cat__Word_Count_Binned_25-30           |   0.00635609 |
#| cat__Cleaned_Primary_Genre_Adult       |   0.00603386 |
#| cat__Word_Count_Binned_15-20           |   0.00577349 |
#| cat__Cleaned_Primary_Genre_Sport       |   0.00572772 |
#| cat__Director_Jon M. Chu               |   0.00543463 |
#| cat__Certificate_PG                    |   0.00534882 |
#

# IMDb theme colors
BACKGROUND = '#000000'  
TEXT = '#f5f5f5'        
HIGHLIGHT = '#f5c518'  

# Set overall seaborn and matplotlib style
sns.set_style("darkgrid")
plt.rcParams.update({
    'axes.facecolor': BACKGROUND,
    'figure.facecolor': BACKGROUND,
    'axes.labelcolor': TEXT,
    'xtick.color': TEXT,
    'ytick.color': TEXT,
    'text.color': TEXT,
    'axes.edgecolor': TEXT,
    'grid.color': '#444444',  # soft gridlines
    'axes.titleweight': 'bold',
    'axes.titlepad': 15,
    'axes.titlesize': 14
})

# 1️⃣ Duration vs IMDb Rating — Sweet spot check
plt.figure(figsize=(8, 5))
sns.scatterplot(x=df['Duration'], y=df['Rating'], alpha=0.4, color='white')  # Data points
sns.lineplot(x='Duration', y='Rating', data=df, estimator='mean', errorbar=None, color=HIGHLIGHT, linewidth=2)
plt.title('IMDb Rating by Duration')
plt.xlabel('Duration (min)')
plt.ylabel('Rating')
plt.grid(True)
plt.tight_layout()
plt.show()

# 2️⃣ Director also being the writer (1 = Yes, 0 = No)
plt.figure(figsize=(6, 4))
ax = sns.barplot(x='Director_Is_Writer', y='Rating', data=df, palette=[TEXT, HIGHLIGHT])
plt.title('Average Rating: Director Also Writer')
plt.ylim(6.0, 8.5)
plt.xlabel('Is Director Writer?')
plt.ylabel('Avg IMDb Rating')
plt.tight_layout()
plt.show()