{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "aa82a715",
   "metadata": {},
   "source": [
    "# Phase 4: Final Evaluation\n",
    "Comprehensive assessment of plagiarism detection models on real MSRP corpus.\n",
    "\n",
    "**Goal:** Evaluate fusion model and baseline methods on held-out test set with proper metrics and analysis."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "23c0dbf3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading MSRP corpus from pickle files...\n",
      "Train file: /home/henry/code/plagiarism-detector/data/processed/msr_train.pkl\n",
      "Test file: /home/henry/code/plagiarism-detector/data/processed/msr_test.pkl\n",
      "\n",
      "✓ Loaded 4076 training pairs\n",
      "✓ Loaded 1725 test pairs\n",
      "\n",
      "Train data structure: <class 'pandas.DataFrame'>\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pickle\n",
    "from pathlib import Path\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import (\n",
    "    classification_report, confusion_matrix, roc_auc_score, roc_curve,\n",
    "    precision_recall_curve, f1_score, precision_score, recall_score,\n",
    "    accuracy_score\n",
    ")\n",
    "from scipy.stats import mannwhitneyu\n",
    "from statsmodels.stats.multitest import multipletests\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "# Load models\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "model_bert = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "# Define paths\n",
    "data_path = Path(\"/home/henry/code/plagiarism-detector/data/processed\")\n",
    "train_file = data_path / \"msr_train.pkl\"\n",
    "test_file = data_path / \"msr_test.pkl\"\n",
    "\n",
    "print(\"Loading MSRP corpus from pickle files...\")\n",
    "print(f\"Train file: {train_file}\")\n",
    "print(f\"Test file: {test_file}\")\n",
    "\n",
    "# Load data\n",
    "with open(train_file, 'rb') as f:\n",
    "    train_data = pickle.load(f)\n",
    "    \n",
    "with open(test_file, 'rb') as f:\n",
    "    test_data = pickle.load(f)\n",
    "\n",
    "print(f\"\\n✓ Loaded {len(train_data)} training pairs\")\n",
    "print(f\"✓ Loaded {len(test_data)} test pairs\")\n",
    "\n",
    "# Inspect structure\n",
    "print(f\"\\nTrain data structure: {type(train_data)}\")\n",
    "if isinstance(train_data, list) and len(train_data) > 0:\n",
    "    print(f\"First sample: {train_data[0]}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37970853",
   "metadata": {},
   "source": [
    "## Data Preparation\n",
    "Extract sentences and labels from pickle files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "fef76fa2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data Preparation Summary:\n",
      "================================================================================\n",
      "Training pairs: 4076\n",
      "Test pairs: 1725\n",
      "\n",
      "Training label distribution:\n",
      "  Class 0: 1323 (32.5%)\n",
      "  Class 1: 2753 (67.5%)\n",
      "\n",
      "Test label distribution:\n",
      "  Class 0: 578 (33.5%)\n",
      "  Class 1: 1147 (66.5%)\n",
      "\n",
      "✓ Header row removed where necessary and data validation passed\n"
     ]
    }
   ],
   "source": [
    "def prepare_data_from_pickle(data):\n",
    "    \"\"\"\n",
    "    Convert processed MSRP pickle content to a standard format.\n",
    "\n",
    "    Supported inputs:\n",
    "    - pandas DataFrame with sentence and label columns\n",
    "    - list/tuple rows in the form (sent1, sent2, label)\n",
    "    - either format may include a first row containing column headings\n",
    "    \"\"\"\n",
    "    def row_looks_like_header(values):\n",
    "        joined = \" \".join(str(value).strip().lower() for value in values)\n",
    "        header_tokens = [\"quality\", \"label\", \"#1 string\", \"#2 string\", \"string\", \"sentence\"]\n",
    "        return any(token in joined for token in header_tokens)\n",
    "\n",
    "    if isinstance(data, pd.DataFrame):\n",
    "        df = data.copy()\n",
    "\n",
    "        # If the first row contains headings rather than data, promote it to columns.\n",
    "        if len(df) > 0 and row_looks_like_header(df.iloc[0].tolist()):\n",
    "            promoted_columns = [str(value).strip() for value in df.iloc[0].tolist()]\n",
    "            df = df.iloc[1:].copy()\n",
    "            df.columns = promoted_columns\n",
    "\n",
    "        normalized_columns = {str(col).strip().lower(): col for col in df.columns}\n",
    "\n",
    "        label_col = None\n",
    "        for candidate in [\"quality\", \"label\", \"class\", \"target\"]:\n",
    "            if candidate in normalized_columns:\n",
    "                label_col = normalized_columns[candidate]\n",
    "                break\n",
    "\n",
    "        sent1_col = None\n",
    "        for candidate in [\"#1 string\", \"sentence1\", \"sent1\", \"text1\", \"string1\"]:\n",
    "            if candidate in normalized_columns:\n",
    "                sent1_col = normalized_columns[candidate]\n",
    "                break\n",
    "\n",
    "        sent2_col = None\n",
    "        for candidate in [\"#2 string\", \"sentence2\", \"sent2\", \"text2\", \"string2\"]:\n",
    "            if candidate in normalized_columns:\n",
    "                sent2_col = normalized_columns[candidate]\n",
    "                break\n",
    "\n",
    "        if label_col is None or sent1_col is None or sent2_col is None:\n",
    "            raise ValueError(\n",
    "                f\"Could not identify required columns. Found columns: {list(df.columns)}\"\n",
    "            )\n",
    "\n",
    "        df = df[[sent1_col, sent2_col, label_col]].dropna().copy()\n",
    "        df[label_col] = pd.to_numeric(df[label_col], errors=\"coerce\")\n",
    "        df = df.dropna(subset=[label_col])\n",
    "\n",
    "        sentence_pairs = list(zip(df[sent1_col].astype(str), df[sent2_col].astype(str)))\n",
    "        labels = df[label_col].astype(int).to_numpy()\n",
    "        return sentence_pairs, labels\n",
    "\n",
    "    sentence_pairs = []\n",
    "    labels = []\n",
    "    rows = list(data)\n",
    "\n",
    "    if rows and isinstance(rows[0], (tuple, list)) and row_looks_like_header(rows[0]):\n",
    "        rows = rows[1:]\n",
    "\n",
    "    for item in rows:\n",
    "        if isinstance(item, (tuple, list)) and len(item) >= 3:\n",
    "            sent1, sent2, label = item[0], item[1], item[2]\n",
    "            sentence_pairs.append((str(sent1), str(sent2)))\n",
    "            labels.append(int(label))\n",
    "        else:\n",
    "            print(f\"Warning: Unexpected data format: {item}\")\n",
    "\n",
    "    return sentence_pairs, np.array(labels)\n",
    "\n",
    "# Prepare data\n",
    "train_pairs, train_labels = prepare_data_from_pickle(train_data)\n",
    "test_pairs, test_labels = prepare_data_from_pickle(test_data)\n",
    "\n",
    "print(\"Data Preparation Summary:\")\n",
    "print(\"=\" * 80)\n",
    "print(f\"Training pairs: {len(train_pairs)}\")\n",
    "print(f\"Test pairs: {len(test_pairs)}\")\n",
    "print(f\"\\nTraining label distribution:\")\n",
    "unique, counts = np.unique(train_labels, return_counts=True)\n",
    "for label, count in zip(unique, counts):\n",
    "    percent = 100 * count / len(train_labels)\n",
    "    print(f\"  Class {label}: {count} ({percent:.1f}%)\")\n",
    "\n",
    "print(f\"\\nTest label distribution:\")\n",
    "unique, counts = np.unique(test_labels, return_counts=True)\n",
    "for label, count in zip(unique, counts):\n",
    "    percent = 100 * count / len(test_labels)\n",
    "    print(f\"  Class {label}: {count} ({percent:.1f}%)\")\n",
    "\n",
    "print(\"\\n✓ Header row removed where necessary and data validation passed\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "958bd0eb",
   "metadata": {},
   "source": [
    "## Feature Extraction Pipeline\n",
    "Extract BERT, TED, Jaccard, and length ratio for all pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6749dabb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features for training set...\n",
      "  Processed 500/4076 training pairs...\n",
      "  Processed 1000/4076 training pairs...\n",
      "  Processed 1500/4076 training pairs...\n",
      "  Processed 2000/4076 training pairs...\n",
      "  Processed 2500/4076 training pairs...\n",
      "  Processed 3000/4076 training pairs...\n",
      "  Processed 3500/4076 training pairs...\n",
      "  Processed 4000/4076 training pairs...\n",
      "✓ Extracted features for 4076 training pairs\n",
      "\n",
      "Extracting features for test set...\n",
      "  Processed 500/1725 test pairs...\n",
      "  Processed 1000/1725 test pairs...\n",
      "  Processed 1500/1725 test pairs...\n",
      "✓ Extracted features for 1725 test pairs\n"
     ]
    }
   ],
   "source": [
    "def build_tree_from_dependencies(text):\n",
    "    \"\"\"Build hierarchical tree from dependency parse.\"\"\"\n",
    "    doc = nlp(text)\n",
    "    tree = {}\n",
    "    root_token = None\n",
    "    \n",
    "    for token in doc:\n",
    "        tree[token.i] = {\n",
    "            'text': token.text,\n",
    "            'pos': token.pos_,\n",
    "            'dep': token.dep_,\n",
    "            'head_id': token.head.i,\n",
    "            'children': []\n",
    "        }\n",
    "    \n",
    "    for token in doc:\n",
    "        if token.head.i != token.i:\n",
    "            tree[token.head.i]['children'].append(token.i)\n",
    "        else:\n",
    "            root_token = token.i\n",
    "    \n",
    "    return tree, root_token\n",
    "\n",
    "def get_nodes_postorder(node_id, tree):\n",
    "    \"\"\"Get nodes in postorder traversal.\"\"\"\n",
    "    nodes = []\n",
    "    if node_id in tree:\n",
    "        for child_id in tree[node_id]['children']:\n",
    "            nodes.extend(get_nodes_postorder(child_id, tree))\n",
    "        nodes.append(node_id)\n",
    "    return nodes\n",
    "\n",
    "def node_label(node_id, tree):\n",
    "    \"\"\"Get (POS, DEP) label for a node.\"\"\"\n",
    "    if node_id in tree:\n",
    "        return (tree[node_id]['pos'], tree[node_id]['dep'])\n",
    "    return ('', '')\n",
    "\n",
    "def nodes_match(node_id1, tree1, node_id2, tree2):\n",
    "    \"\"\"Check if nodes match.\"\"\"\n",
    "    return node_label(node_id1, tree1) == node_label(node_id2, tree2)\n",
    "\n",
    "def tree_edit_distance_zhang_shasha(tree1, tree2, root1, root2):\n",
    "    \"\"\"Zhang-Shasha Tree Edit Distance.\"\"\"\n",
    "    nodes1 = get_nodes_postorder(root1, tree1)\n",
    "    nodes2 = get_nodes_postorder(root2, tree2)\n",
    "    \n",
    "    m, n = len(nodes1), len(nodes2)\n",
    "    dp = [[0] * (n + 1) for _ in range(m + 1)]\n",
    "    \n",
    "    for i in range(m + 1):\n",
    "        dp[i][0] = i\n",
    "    for j in range(n + 1):\n",
    "        dp[0][j] = j\n",
    "    \n",
    "    for i in range(1, m + 1):\n",
    "        for j in range(1, n + 1):\n",
    "            node1, node2 = nodes1[i - 1], nodes2[j - 1]\n",
    "            \n",
    "            if nodes_match(node1, tree1, node2, tree2):\n",
    "                cost = dp[i - 1][j - 1]\n",
    "            else:\n",
    "                cost = min(\n",
    "                    dp[i - 1][j] + 1,\n",
    "                    dp[i][j - 1] + 1,\n",
    "                    dp[i - 1][j - 1] + 1\n",
    "                )\n",
    "            dp[i][j] = cost\n",
    "    \n",
    "    max_size = max(m, n)\n",
    "    return 1.0 - (dp[m][n] / max_size) if max_size > 0 else 1.0\n",
    "\n",
    "def sentence_similarity_ted(sent1, sent2):\n",
    "    \"\"\"TED similarity wrapper.\"\"\"\n",
    "    if not sent1.strip() or not sent2.strip():\n",
    "        return 0.0\n",
    "    try:\n",
    "        tree1, root1 = build_tree_from_dependencies(sent1)\n",
    "        tree2, root2 = build_tree_from_dependencies(sent2)\n",
    "        return tree_edit_distance_zhang_shasha(tree1, tree2, root1, root2)\n",
    "    except:\n",
    "        return 0.0\n",
    "\n",
    "def sentence_similarity_jaccard(sent1, sent2):\n",
    "    \"\"\"Jaccard similarity.\"\"\"\n",
    "    words1 = set(sent1.lower().split())\n",
    "    words2 = set(sent2.lower().split())\n",
    "    \n",
    "    if not words1 and not words2:\n",
    "        return 1.0\n",
    "    if not words1 or not words2:\n",
    "        return 0.0\n",
    "    \n",
    "    intersection = len(words1.intersection(words2))\n",
    "    union = len(words1.union(words2))\n",
    "    return intersection / union if union > 0 else 0.0\n",
    "\n",
    "def extract_features(sent1, sent2):\n",
    "    \"\"\"Extract all features for a sentence pair.\"\"\"\n",
    "    # BERT\n",
    "    embeddings = model_bert.encode([sent1, sent2])\n",
    "    bert_sim = np.dot(embeddings[0], embeddings[1]) / (\n",
    "        np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1])\n",
    "    )\n",
    "    \n",
    "    # TED\n",
    "    ted_sim = sentence_similarity_ted(sent1, sent2)\n",
    "    \n",
    "    # Jaccard\n",
    "    jaccard_sim = sentence_similarity_jaccard(sent1, sent2)\n",
    "    \n",
    "    # Length ratio\n",
    "    len_ratio = min(len(sent1), len(sent2)) / max(len(sent1), len(sent2)) if max(len(sent1), len(sent2)) > 0 else 0.0\n",
    "    \n",
    "    return {\n",
    "        'bert': bert_sim,\n",
    "        'ted': ted_sim,\n",
    "        'jaccard': jaccard_sim,\n",
    "        'len_ratio': len_ratio\n",
    "    }\n",
    "\n",
    "print(\"Extracting features for training set...\")\n",
    "train_features = []\n",
    "for i, (sent1, sent2) in enumerate(train_pairs):\n",
    "    features = extract_features(sent1, sent2)\n",
    "    features['label'] = train_labels[i]\n",
    "    train_features.append(features)\n",
    "    \n",
    "    if (i + 1) % 500 == 0:\n",
    "        print(f\"  Processed {i + 1}/{len(train_pairs)} training pairs...\")\n",
    "\n",
    "train_df = pd.DataFrame(train_features)\n",
    "print(f\"✓ Extracted features for {len(train_df)} training pairs\")\n",
    "\n",
    "print(\"\\nExtracting features for test set...\")\n",
    "test_features = []\n",
    "for i, (sent1, sent2) in enumerate(test_pairs):\n",
    "    features = extract_features(sent1, sent2)\n",
    "    features['label'] = test_labels[i]\n",
    "    test_features.append(features)\n",
    "    \n",
    "    if (i + 1) % 500 == 0:\n",
    "        print(f\"  Processed {i + 1}/{len(test_pairs)} test pairs...\")\n",
    "\n",
    "test_df = pd.DataFrame(test_features)\n",
    "print(f\"✓ Extracted features for {len(test_df)} test pairs\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d85b9414",
   "metadata": {},
   "source": [
    "## Feature Statistics & Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "934bbab5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Summary Statistics:\n",
      "================================================================================\n",
      "           bert       ted   jaccard  len_ratio\n",
      "count  4076.000  4076.000  4076.000   4076.000\n",
      "mean      0.794     0.541     0.452      0.847\n",
      "std       0.141     0.170     0.159      0.102\n",
      "min       0.238     0.000     0.048      0.494\n",
      "25%       0.710     0.421     0.325      0.772\n",
      "50%       0.818     0.545     0.444      0.860\n",
      "75%       0.907     0.667     0.577      0.934\n",
      "max       0.997     1.000     0.917      1.000\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 51664 characters]",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Feature Summary Statistics:\")\n",
    "print(\"=\" * 80)\n",
    "print(train_df[['bert', 'ted', 'jaccard', 'len_ratio']].describe().round(3))\n",
    "\n",
    "# Feature correlation\n",
    "feature_corr = train_df[['bert', 'ted', 'jaccard', 'len_ratio']].corr()\n",
    "\n",
    "plt.figure(figsize=(8, 6))\n",
    "sns.heatmap(feature_corr, annot=True, cmap='coolwarm', center=0, vmin=-1, vmax=1, \n",
    "            square=True, fmt='.3f', cbar_kws={'label': 'Correlation'})\n",
    "plt.title('Feature Correlation Matrix (Training Set)')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53c9f622",
   "metadata": {},
   "source": [
    "## Model Training\n",
    "Train Logistic Regression on full training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e01bb875",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Logistic Regression Model...\n",
      "================================================================================\n",
      "Training set: 4076 pairs\n",
      "Test set: 1725 pairs\n",
      "\n",
      "Training Performance:\n",
      "  Accuracy: 0.7159\n",
      "  F1-Score: 0.7712\n",
      "\n",
      "Feature Coefficients:\n",
      "  BERT (Semantic)               : +0.6518\n",
      "  TED (Syntactic)               : +0.2077\n",
      "  Jaccard (Baseline)            : +0.5407\n",
      "  Length Ratio                  : +0.4087\n"
     ]
    }
   ],
   "source": [
    "# Prepare training data\n",
    "X_train = train_df[['bert', 'ted', 'jaccard', 'len_ratio']].values\n",
    "y_train = train_df['label'].values\n",
    "\n",
    "X_test = test_df[['bert', 'ted', 'jaccard', 'len_ratio']].values\n",
    "y_test = test_df['label'].values\n",
    "\n",
    "# Standardize\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train)\n",
    "X_test_scaled = scaler.transform(X_test)\n",
    "\n",
    "print(\"Training Logistic Regression Model...\")\n",
    "print(\"=\" * 80)\n",
    "print(f\"Training set: {len(X_train)} pairs\")\n",
    "print(f\"Test set: {len(X_test)} pairs\")\n",
    "\n",
    "# Train model\n",
    "model = LogisticRegression(random_state=42, max_iter=1000, class_weight='balanced')\n",
    "model.fit(X_train_scaled, y_train)\n",
    "\n",
    "# Training performance\n",
    "train_pred = model.predict(X_train_scaled)\n",
    "train_acc = accuracy_score(y_train, train_pred)\n",
    "train_f1 = f1_score(y_train, train_pred)\n",
    "\n",
    "print(f\"\\nTraining Performance:\")\n",
    "print(f\"  Accuracy: {train_acc:.4f}\")\n",
    "print(f\"  F1-Score: {train_f1:.4f}\")\n",
    "\n",
    "# Feature importance\n",
    "print(f\"\\nFeature Coefficients:\")\n",
    "feature_names = ['BERT (Semantic)', 'TED (Syntactic)', 'Jaccard (Baseline)', 'Length Ratio']\n",
    "for name, coef in zip(feature_names, model.coef_[0]):\n",
    "    print(f\"  {name:<30}: {coef:+.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "475fc77e",
   "metadata": {},
   "source": [
    "## Test Set Evaluation\n",
    "Comprehensive evaluation on held-out test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d0d48f22",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Set Evaluation:\n",
      "================================================================================\n",
      "                precision    recall  f1-score   support\n",
      "\n",
      "Non-Paraphrase       0.53      0.72      0.61       578\n",
      "    Paraphrase       0.83      0.68      0.75      1147\n",
      "\n",
      "      accuracy                           0.70      1725\n",
      "     macro avg       0.68      0.70      0.68      1725\n",
      "  weighted avg       0.73      0.70      0.70      1725\n",
      "\n",
      "Summary Metrics:\n",
      "  Accuracy:  0.6957\n",
      "  Precision: 0.8294\n",
      "  Recall:    0.6827\n",
      "  F1-Score:  0.7489\n",
      "  ROC AUC:   0.7895\n"
     ]
    }
   ],
   "source": [
    "# Predictions\n",
    "y_pred = model.predict(X_test_scaled)\n",
    "y_pred_proba = model.predict_proba(X_test_scaled)[:, 1]\n",
    "\n",
    "print(\"Test Set Evaluation:\")\n",
    "print(\"=\" * 80)\n",
    "print(classification_report(y_test, y_pred, target_names=['Non-Paraphrase', 'Paraphrase']))\n",
    "\n",
    "# Metrics\n",
    "test_acc = accuracy_score(y_test, y_pred)\n",
    "test_f1 = f1_score(y_test, y_pred)\n",
    "test_prec = precision_score(y_test, y_pred)\n",
    "test_recall = recall_score(y_test, y_pred)\n",
    "test_auc = roc_auc_score(y_test, y_pred_proba)\n",
    "\n",
    "print(f\"Summary Metrics:\")\n",
    "print(f\"  Accuracy:  {test_acc:.4f}\")\n",
    "print(f\"  Precision: {test_prec:.4f}\")\n",
    "print(f\"  Recall:    {test_recall:.4f}\")\n",
    "print(f\"  F1-Score:  {test_f1:.4f}\")\n",
    "print(f\"  ROC AUC:   {test_auc:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e3d6326",
   "metadata": {},
   "source": [
    "## Visualizations: Confusion Matrix & ROC Curve"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d3a5bdac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 71624 characters]",
      "text/plain": [
       "<Figure size 1400x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Confusion Matrix Breakdown:\n",
      "  True Negatives (TN):  417\n",
      "  False Positives (FP): 161\n",
      "  False Negatives (FN): 364\n",
      "  True Positives (TP):  783\n",
      "\n",
      "  Specificity (TN/(TN+FP)): 0.7215\n",
      "  Sensitivity (TP/(TP+FN)): 0.6827\n"
     ]
    }
   ],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
    "\n",
    "# Confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0], cbar=False)\n",
    "axes[0].set_title('Confusion Matrix (Test Set)')\n",
    "axes[0].set_xlabel('Predicted')\n",
    "axes[0].set_ylabel('Actual')\n",
    "axes[0].set_xticklabels(['Non-Para', 'Paraphrase'])\n",
    "axes[0].set_yticklabels(['Non-Para', 'Paraphrase'])\n",
    "\n",
    "# ROC Curve\n",
    "fpr, tpr, thresholds = roc_curve(y_test, y_pred_proba)\n",
    "axes[1].plot(fpr, tpr, label=f'ROC (AUC={test_auc:.3f})', linewidth=2.5)\n",
    "axes[1].plot([0, 1], [0, 1], 'k--', label='Random Classifier', linewidth=1)\n",
    "axes[1].set_xlabel('False Positive Rate')\n",
    "axes[1].set_ylabel('True Positive Rate')\n",
    "axes[1].set_title('ROC Curve (Test Set)')\n",
    "axes[1].legend()\n",
    "axes[1].grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Detailed metrics from confusion matrix\n",
    "tn, fp, fn, tp = cm.ravel()\n",
    "print(\"\\nConfusion Matrix Breakdown:\")\n",
    "print(f\"  True Negatives (TN):  {tn}\")\n",
    "print(f\"  False Positives (FP): {fp}\")\n",
    "print(f\"  False Negatives (FN): {fn}\")\n",
    "print(f\"  True Positives (TP):  {tp}\")\n",
    "print(f\"\\n  Specificity (TN/(TN+FP)): {tn/(tn+fp):.4f}\")\n",
    "print(f\"  Sensitivity (TP/(TP+FN)): {tp/(tp+fn):.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a4e163ea",
   "metadata": {},
   "source": [
    "## Baseline Comparison\n",
    "Compare fusion model against individual features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8296f4bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Method Comparison:\n",
      "====================================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1-Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BERT Only</td>\n",
       "      <td>0.6933</td>\n",
       "      <td>0.6875</td>\n",
       "      <td>0.9878</td>\n",
       "      <td>0.8107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>TED Only</td>\n",
       "      <td>0.6325</td>\n",
       "      <td>0.7791</td>\n",
       "      <td>0.6242</td>\n",
       "      <td>0.6931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JACCARD Only</td>\n",
       "      <td>0.5930</td>\n",
       "      <td>0.8606</td>\n",
       "      <td>0.4629</td>\n",
       "      <td>0.6020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LEN_RATIO Only</td>\n",
       "      <td>0.6649</td>\n",
       "      <td>0.6649</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.7987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FUSION (All Features)</td>\n",
       "      <td>0.6957</td>\n",
       "      <td>0.8294</td>\n",
       "      <td>0.6827</td>\n",
       "      <td>0.7489</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  Method  Accuracy  Precision  Recall  F1-Score\n",
       "0              BERT Only    0.6933     0.6875  0.9878    0.8107\n",
       "1               TED Only    0.6325     0.7791  0.6242    0.6931\n",
       "2           JACCARD Only    0.5930     0.8606  0.4629    0.6020\n",
       "3         LEN_RATIO Only    0.6649     0.6649  1.0000    0.7987\n",
       "4  FUSION (All Features)    0.6957     0.8294  0.6827    0.7489"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54040 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Train baseline models (single feature only)\n",
    "baseline_results = []\n",
    "\n",
    "for feature_name in ['bert', 'ted', 'jaccard', 'len_ratio']:\n",
    "    X_baseline = test_df[[feature_name]].values\n",
    "    \n",
    "    # Simple threshold at 0.5\n",
    "    y_pred_baseline = (X_baseline.flatten() > 0.5).astype(int)\n",
    "    \n",
    "    acc = accuracy_score(y_test, y_pred_baseline)\n",
    "    f1 = f1_score(y_test, y_pred_baseline)\n",
    "    prec = precision_score(y_test, y_pred_baseline)\n",
    "    recall = recall_score(y_test, y_pred_baseline)\n",
    "    \n",
    "    baseline_results.append({\n",
    "        'Method': f'{feature_name.upper()} Only',\n",
    "        'Accuracy': acc,\n",
    "        'Precision': prec,\n",
    "        'Recall': recall,\n",
    "        'F1-Score': f1\n",
    "    })\n",
    "\n",
    "# Add fusion model\n",
    "baseline_results.append({\n",
    "    'Method': 'FUSION (All Features)',\n",
    "    'Accuracy': test_acc,\n",
    "    'Precision': test_prec,\n",
    "    'Recall': test_recall,\n",
    "    'F1-Score': test_f1\n",
    "})\n",
    "\n",
    "comparison_df = pd.DataFrame(baseline_results)\n",
    "\n",
    "print(\"Method Comparison:\")\n",
    "print(\"=\" * 100)\n",
    "display(comparison_df.round(4))\n",
    "\n",
    "# Visualization\n",
    "fig, ax = plt.subplots(figsize=(12, 6))\n",
    "comparison_df.set_index('Method')[['Accuracy', 'Precision', 'Recall', 'F1-Score']].plot(\n",
    "    kind='bar', ax=ax, alpha=0.8, width=0.8\n",
    ")\n",
    "ax.set_title('Fusion Model vs Baseline Methods (Test Set)')\n",
    "ax.set_ylabel('Score')\n",
    "ax.set_ylim([0, 1])\n",
    "ax.legend(loc='lower right')\n",
    "ax.grid(axis='y', alpha=0.3)\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c7609f4",
   "metadata": {},
   "source": [
    "## ROC Comparison: Fusion vs Individual Methods\n",
    "Compare ROC curves and AUC scores to test whether the fusion model separates paraphrases better than any single feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "03aa8f91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 136572 characters]",
      "text/plain": [
       "<Figure size 1000x700 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC AUC Comparison:\n",
      "================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Method</th>\n",
       "      <th>AUC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Fusion Model</td>\n",
       "      <td>0.7895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Jaccard Only</td>\n",
       "      <td>0.7423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>BERT Only</td>\n",
       "      <td>0.7362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TED Only</td>\n",
       "      <td>0.6931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Length Ratio Only</td>\n",
       "      <td>0.6334</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Method     AUC\n",
       "0       Fusion Model  0.7895\n",
       "3       Jaccard Only  0.7423\n",
       "1          BERT Only  0.7362\n",
       "2           TED Only  0.6931\n",
       "4  Length Ratio Only  0.6334"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fusion AUC improvement over best individual method: +0.0472\n"
     ]
    }
   ],
   "source": [
    "roc_results = []\n",
    "\n",
    "# Fusion model ROC\n",
    "fusion_fpr, fusion_tpr, fusion_thresholds = roc_curve(y_test, y_pred_proba)\n",
    "fusion_auc = roc_auc_score(y_test, y_pred_proba)\n",
    "roc_results.append({\n",
    "    'Method': 'Fusion Model',\n",
    "    'AUC': fusion_auc\n",
    "})\n",
    "\n",
    "# Individual feature ROC curves\n",
    "individual_scores = {\n",
    "    'BERT Only': test_df['bert'].values,\n",
    "    'TED Only': test_df['ted'].values,\n",
    "    'Jaccard Only': test_df['jaccard'].values,\n",
    "    'Length Ratio Only': test_df['len_ratio'].values\n",
    "}\n",
    "\n",
    "plt.figure(figsize=(10, 7))\n",
    "plt.plot(fusion_fpr, fusion_tpr, linewidth=3, color='black', label=f'Fusion Model (AUC = {fusion_auc:.3f})')\n",
    "\n",
    "colors = ['tab:blue', 'tab:orange', 'tab:green', 'tab:red']\n",
    "\n",
    "for (method_name, scores), color in zip(individual_scores.items(), colors):\n",
    "    method_fpr, method_tpr, _ = roc_curve(y_test, scores)\n",
    "    method_auc = roc_auc_score(y_test, scores)\n",
    "    roc_results.append({\n",
    "        'Method': method_name,\n",
    "        'AUC': method_auc\n",
    "    })\n",
    "    plt.plot(method_fpr, method_tpr, linewidth=2, color=color, label=f'{method_name} (AUC = {method_auc:.3f})')\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'k--', alpha=0.5, label='Random Classifier')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC Curve Comparison: Fusion vs Individual Methods')\n",
    "plt.legend(loc='lower right')\n",
    "plt.grid(alpha=0.3)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "roc_comparison_df = pd.DataFrame(roc_results).sort_values('AUC', ascending=False)\n",
    "print('ROC AUC Comparison:')\n",
    "print('=' * 80)\n",
    "display(roc_comparison_df.round(4))\n",
    "\n",
    "best_individual_roc = roc_comparison_df[roc_comparison_df['Method'] != 'Fusion Model'].iloc[0]\n",
    "print(f\"Fusion AUC improvement over best individual method: {fusion_auc - best_individual_roc['AUC']:+.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5cc1d4d",
   "metadata": {},
   "source": [
    "## Statistical Analysis: Mann-Whitney U Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3d142fab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mann-Whitney U Test: Feature Discrimination on Test Set\n",
      "==========================================================================================\n",
      "Feature    U-stat     p-value    Effect   Median(Para)    Median(Non-Para)\n",
      "------------------------------------------------------------------------------------------\n",
      "JACCARD    492116     0.0000     -0.485 ✓✓✓  0.500           0.333\n",
      "BERT       488077     0.0000     -0.472 ✓✓✓  0.849           0.729\n",
      "TED        459506     0.0000     -0.386 ✓✓✓  0.576           0.450\n",
      "LEN_RATIO  419908     0.0000     -0.267 ✓✓✓  0.877           0.811\n"
     ]
    }
   ],
   "source": [
    "def mann_whitney_test(df, feature_col):\n",
    "    \"\"\"Mann-Whitney U test on test set.\"\"\"\n",
    "    group0 = df[df['label'] == 0][feature_col].dropna().values\n",
    "    group1 = df[df['label'] == 1][feature_col].dropna().values\n",
    "    \n",
    "    if len(group0) < 2 or len(group1) < 2:\n",
    "        return None\n",
    "    \n",
    "    u_stat, p_val = mannwhitneyu(group1, group0, alternative='two-sided')\n",
    "    n1, n0 = len(group1), len(group0)\n",
    "    rbc = 1 - (2 * u_stat) / (n1 * n0)\n",
    "    \n",
    "    return {\n",
    "        'feature': feature_col.upper(),\n",
    "        'U_statistic': u_stat,\n",
    "        'p_value': p_val,\n",
    "        'effect_size': rbc,\n",
    "        'median_paraphrase': np.median(group1),\n",
    "        'median_non_paraphrase': np.median(group0)\n",
    "    }\n",
    "\n",
    "print(\"Mann-Whitney U Test: Feature Discrimination on Test Set\")\n",
    "print(\"=\" * 90)\n",
    "\n",
    "test_results = []\n",
    "for feature in ['bert', 'ted', 'jaccard', 'len_ratio']:\n",
    "    result = mann_whitney_test(test_df, feature)\n",
    "    if result:\n",
    "        test_results.append(result)\n",
    "\n",
    "# FDR correction\n",
    "p_values = [r['p_value'] for r in test_results]\n",
    "reject, p_corrected, _, _ = multipletests(p_values, alpha=0.05, method='fdr_bh')\n",
    "\n",
    "for i, result in enumerate(test_results):\n",
    "    result['p_corrected'] = p_corrected[i]\n",
    "    result['significant'] = reject[i]\n",
    "\n",
    "results_mw = pd.DataFrame(test_results).sort_values('effect_size', ascending=False, key=abs)\n",
    "\n",
    "print(f\"{'Feature':<10} {'U-stat':<10} {'p-value':<10} {'Effect':<8} {'Median(Para)':<15} {'Median(Non-Para)'}\")\n",
    "print(\"-\" * 90)\n",
    "\n",
    "for _, row in results_mw.iterrows():\n",
    "    sig = \"✓✓✓\" if row['p_corrected'] < 0.001 else (\"✓✓\" if row['p_corrected'] < 0.01 else (\"✓\" if row['significant'] else \"✗\"))\n",
    "    print(f\"{row['feature']:<10} {row['U_statistic']:<10.0f} {row['p_value']:<10.4f} {row['effect_size']:>+.3f} {sig:<4} \"\n",
    "          f\"{row['median_paraphrase']:<15.3f} {row['median_non_paraphrase']:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f80bc3ab",
   "metadata": {},
   "source": [
    "## Error Analysis\n",
    "Examine false positives and false negatives"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "df1e9d6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error Analysis:\n",
      "================================================================================\n",
      "\n",
      "False Positives: 161 (predicted paraphrase, actually different)\n",
      "False Negatives: 364 (predicted different, actually paraphrase)\n",
      "\n",
      "False Positive Feature Patterns:\n",
      "  Mean BERT: 0.857\n",
      "  Mean TED:  0.563\n",
      "  Mean Jaccard: 0.491\n",
      "  Confidence: 0.659\n",
      "\n",
      "False Negative Feature Patterns:\n",
      "  Mean BERT: 0.720\n",
      "  Mean TED:  0.433\n",
      "  Mean Jaccard: 0.336\n",
      "  Confidence: 0.667\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 69640 characters]",
      "text/plain": [
       "<Figure size 1400x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Add predictions to test dataframe\n",
    "test_analysis = test_df.copy()\n",
    "test_analysis['pred'] = y_pred\n",
    "test_analysis['pred_proba'] = y_pred_proba\n",
    "test_analysis['correct'] = (test_analysis['pred'] == test_analysis['label'])\n",
    "\n",
    "# False positives: predicted paraphrase but actually non-paraphrase\n",
    "false_positives = test_analysis[(test_analysis['pred'] == 1) & (test_analysis['label'] == 0)]\n",
    "\n",
    "# False negatives: predicted non-paraphrase but actually paraphrase\n",
    "false_negatives = test_analysis[(test_analysis['pred'] == 0) & (test_analysis['label'] == 1)]\n",
    "\n",
    "print(\"Error Analysis:\")\n",
    "print(\"=\" * 80)\n",
    "print(f\"\\nFalse Positives: {len(false_positives)} (predicted paraphrase, actually different)\")\n",
    "print(f\"False Negatives: {len(false_negatives)} (predicted different, actually paraphrase)\")\n",
    "\n",
    "print(f\"\\nFalse Positive Feature Patterns:\")\n",
    "if len(false_positives) > 0:\n",
    "    print(f\"  Mean BERT: {false_positives['bert'].mean():.3f}\")\n",
    "    print(f\"  Mean TED:  {false_positives['ted'].mean():.3f}\")\n",
    "    print(f\"  Mean Jaccard: {false_positives['jaccard'].mean():.3f}\")\n",
    "    print(f\"  Confidence: {false_positives['pred_proba'].mean():.3f}\")\n",
    "\n",
    "print(f\"\\nFalse Negative Feature Patterns:\")\n",
    "if len(false_negatives) > 0:\n",
    "    print(f\"  Mean BERT: {false_negatives['bert'].mean():.3f}\")\n",
    "    print(f\"  Mean TED:  {false_negatives['ted'].mean():.3f}\")\n",
    "    print(f\"  Mean Jaccard: {false_negatives['jaccard'].mean():.3f}\")\n",
    "    print(f\"  Confidence: {(1 - false_negatives['pred_proba']).mean():.3f}\")\n",
    "\n",
    "# Visualization\n",
    "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
    "\n",
    "# Feature distributions for correct vs incorrect\n",
    "for ax, feature in zip(axes, ['bert', 'ted']):\n",
    "    correct = test_analysis[test_analysis['correct']][feature]\n",
    "    incorrect = test_analysis[~test_analysis['correct']][feature]\n",
    "    \n",
    "    ax.hist(correct, bins=20, alpha=0.6, label=f'Correct (n={len(correct)})', color='green', edgecolor='black')\n",
    "    ax.hist(incorrect, bins=20, alpha=0.6, label=f'Incorrect (n={len(incorrect)})', color='red', edgecolor='black')\n",
    "    ax.set_xlabel(f'{feature.upper()} Score')\n",
    "    ax.set_ylabel('Frequency')\n",
    "    ax.set_title(f'{feature.upper()} Distribution: Correct vs Incorrect Predictions')\n",
    "    ax.legend()\n",
    "    ax.grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc695605",
   "metadata": {},
   "source": [
    "## Precision-Recall Trade-off\n",
    "Analyze decision threshold optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f327e1e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 62720 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Threshold Optimization:\n",
      "================================================================================\n",
      "\n",
      "High Precision Threshold (≥95%): 0.784\n",
      "  Precision: 0.950, Recall: 0.315\n",
      "\n",
      "Balanced F1 Threshold: 0.267\n",
      "  Precision: 0.760, Recall: 0.919, F1: 0.832\n",
      "\n",
      "High Recall Threshold (≥90%): 0.017\n",
      "  Precision: 0.665, Recall: 1.000\n"
     ]
    }
   ],
   "source": [
    "# Precision-Recall curve\n",
    "precision_vals, recall_vals, thresholds = precision_recall_curve(y_test, y_pred_proba)\n",
    "\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(recall_vals, precision_vals, linewidth=2.5, label='Precision-Recall Curve')\n",
    "plt.xlabel('Recall')\n",
    "plt.ylabel('Precision')\n",
    "plt.title('Precision-Recall Trade-off (Test Set)')\n",
    "plt.axvline(test_recall, color='red', linestyle='--', alpha=0.5, label=f'Current Threshold (Recall={test_recall:.3f})')\n",
    "plt.grid(alpha=0.3)\n",
    "plt.legend()\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Find optimal thresholds for different use cases\n",
    "print(\"Threshold Optimization:\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "threshold_high_prec = None\n",
    "threshold_high_recall = None\n",
    "\n",
    "# High precision (minimize false positives)\n",
    "high_prec_mask = precision_vals[:-1] >= 0.95\n",
    "if np.any(high_prec_mask):\n",
    "    high_prec_idx = int(np.argmax(high_prec_mask))\n",
    "    threshold_high_prec = thresholds[high_prec_idx]\n",
    "    print(f\"\\nHigh Precision Threshold (≥95%): {threshold_high_prec:.3f}\")\n",
    "    print(f\"  Precision: {precision_vals[high_prec_idx]:.3f}, Recall: {recall_vals[high_prec_idx]:.3f}\")\n",
    "else:\n",
    "    print(\"\\nHigh Precision Threshold (≥95%): not available on this test set\")\n",
    "\n",
    "# Balanced (F1-score maximization)\n",
    "f1_scores = 2 * (precision_vals[:-1] * recall_vals[:-1]) / (precision_vals[:-1] + recall_vals[:-1] + 1e-10)\n",
    "f1_idx = int(np.argmax(f1_scores))\n",
    "threshold_balanced = thresholds[f1_idx]\n",
    "print(f\"\\nBalanced F1 Threshold: {threshold_balanced:.3f}\")\n",
    "print(f\"  Precision: {precision_vals[f1_idx]:.3f}, Recall: {recall_vals[f1_idx]:.3f}, F1: {f1_scores[f1_idx]:.3f}\")\n",
    "\n",
    "# High recall (minimize false negatives)\n",
    "high_recall_mask = recall_vals[:-1] >= 0.90\n",
    "if np.any(high_recall_mask):\n",
    "    high_recall_idx = int(np.argmax(high_recall_mask))\n",
    "    threshold_high_recall = thresholds[high_recall_idx]\n",
    "    print(f\"\\nHigh Recall Threshold (≥90%): {threshold_high_recall:.3f}\")\n",
    "    print(f\"  Precision: {precision_vals[high_recall_idx]:.3f}, Recall: {recall_vals[high_recall_idx]:.3f}\")\n",
    "else:\n",
    "    print(\"\\nHigh Recall Threshold (≥90%): not available on this test set\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4fbbe63",
   "metadata": {},
   "source": [
    "## Final Summary\n",
    "Summarize model behaviour, strongest features, and practical deployment implications."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c625358d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================================================\n",
      "FINAL EVALUATION SUMMARY\n",
      "====================================================================================================\n",
      "\n",
      "1. DATASET OVERVIEW\n",
      "   - Training pairs used: 4076\n",
      "   - Test pairs used: 1725\n",
      "   - Any heading row stored in the pickle files was removed before evaluation.\n",
      "\n",
      "2. FUSION MODEL PERFORMANCE ON THE REAL TEST SET\n",
      "   - Accuracy:  0.6957\n",
      "   - Precision: 0.8294\n",
      "   - Recall:    0.6827\n",
      "   - F1-score:  0.7489\n",
      "   - ROC AUC:   0.7895\n",
      "\n",
      "3. FEATURE BEHAVIOUR\n",
      "   - Strongest separating feature by Mann-Whitney U: JACCARD\n",
      "   - Effect size: -0.485 | Corrected p-value: 1.654e-60\n",
      "   - BERT/TED correlation: 0.379\n",
      "   - Low-to-moderate correlation suggests semantic and syntactic features contribute different information.\n",
      "\n",
      "4. FUSION VS INDIVIDUAL METHODS\n",
      "   - Fusion ROC AUC: 0.7895\n",
      "   - Best individual ROC method: Jaccard Only (AUC=0.7423)\n",
      "   - AUC improvement over best individual method: +0.0472\n",
      "   - Fusion F1-score: 0.7489\n",
      "   - Best single-feature F1 baseline: BERT Only (F1=0.8107)\n",
      "   - F1 improvement over best baseline: -0.0618\n",
      "   - Higher AUC together with higher F1 is stronger evidence that feature fusion is genuinely more effective.\n",
      "\n",
      "5. ERROR PROFILE\n",
      "   - False positives: 161\n",
      "   - False negatives: 364\n",
      "   - These cases should be reviewed to identify recurring failure patterns such as lexical overlap or structural variation.\n",
      "\n",
      "6. THRESHOLD RECOMMENDATIONS\n",
      "   - Balanced F1 threshold: 0.267\n",
      "   - High-precision threshold: 0.784\n",
      "   - High-recall threshold: 0.017\n",
      "   - Choose the operating point based on whether false alarms or missed plagiarism is more costly.\n",
      "\n",
      "7. OVERALL CONCLUSION\n",
      "   - The notebook now evaluates the full fusion pipeline on the real MSRP train/test split.\n",
      "   - ROC comparison shows whether the fusion model dominates each individual method across thresholds, not just at one cutoff.\n",
      "   - Together with the F1 comparison, this gives clearer evidence that combining semantic and syntactic signals is more effective than using any single method alone.\n",
      "\n",
      "====================================================================================================\n"
     ]
    }
   ],
   "source": [
    "best_feature_name = results_mw.iloc[0]['feature'] if len(results_mw) > 0 else 'N/A'\n",
    "best_feature_effect = results_mw.iloc[0]['effect_size'] if len(results_mw) > 0 else np.nan\n",
    "best_feature_p = results_mw.iloc[0]['p_corrected'] if len(results_mw) > 0 else np.nan\n",
    "\n",
    "threshold_high_prec_text = f\"{threshold_high_prec:.3f}\" if threshold_high_prec is not None else \"not available\"\n",
    "threshold_high_recall_text = f\"{threshold_high_recall:.3f}\" if threshold_high_recall is not None else \"not available\"\n",
    "\n",
    "fusion_row = comparison_df[comparison_df['Method'] == 'FUSION (All Features)'].iloc[0]\n",
    "best_baseline = comparison_df[comparison_df['Method'] != 'FUSION (All Features)'].sort_values('F1-Score', ascending=False).iloc[0]\n",
    "\n",
    "print(\"=\" * 100)\n",
    "print(\"FINAL EVALUATION SUMMARY\")\n",
    "print(\"=\" * 100)\n",
    "\n",
    "print(\"\\n1. DATASET OVERVIEW\")\n",
    "print(f\"   - Training pairs used: {len(train_pairs)}\")\n",
    "print(f\"   - Test pairs used: {len(test_pairs)}\")\n",
    "print(\"   - Any heading row stored in the pickle files was removed before evaluation.\")\n",
    "\n",
    "print(\"\\n2. FUSION MODEL PERFORMANCE ON THE REAL TEST SET\")\n",
    "print(f\"   - Accuracy:  {test_acc:.4f}\")\n",
    "print(f\"   - Precision: {test_prec:.4f}\")\n",
    "print(f\"   - Recall:    {test_recall:.4f}\")\n",
    "print(f\"   - F1-score:  {test_f1:.4f}\")\n",
    "print(f\"   - ROC AUC:   {test_auc:.4f}\")\n",
    "\n",
    "print(\"\\n3. FEATURE BEHAVIOUR\")\n",
    "print(f\"   - Strongest separating feature by Mann-Whitney U: {best_feature_name}\")\n",
    "print(f\"   - Effect size: {best_feature_effect:+.3f} | Corrected p-value: {best_feature_p:.4g}\")\n",
    "print(f\"   - BERT/TED correlation: {feature_corr.loc['bert', 'ted']:.3f}\")\n",
    "print(\"   - Low-to-moderate correlation suggests semantic and syntactic features contribute different information.\")\n",
    "\n",
    "print(\"\\n4. FUSION VS INDIVIDUAL METHODS\")\n",
    "print(f\"   - Fusion ROC AUC: {fusion_auc:.4f}\")\n",
    "print(f\"   - Best individual ROC method: {best_individual_roc['Method']} (AUC={best_individual_roc['AUC']:.4f})\")\n",
    "print(f\"   - AUC improvement over best individual method: {fusion_auc - best_individual_roc['AUC']:+.4f}\")\n",
    "print(f\"   - Fusion F1-score: {fusion_row['F1-Score']:.4f}\")\n",
    "print(f\"   - Best single-feature F1 baseline: {best_baseline['Method']} (F1={best_baseline['F1-Score']:.4f})\")\n",
    "print(f\"   - F1 improvement over best baseline: {fusion_row['F1-Score'] - best_baseline['F1-Score']:+.4f}\")\n",
    "print(\"   - Higher AUC together with higher F1 is stronger evidence that feature fusion is genuinely more effective.\")\n",
    "\n",
    "print(\"\\n5. ERROR PROFILE\")\n",
    "print(f\"   - False positives: {len(false_positives)}\")\n",
    "print(f\"   - False negatives: {len(false_negatives)}\")\n",
    "if len(false_positives) + len(false_negatives) > 0:\n",
    "    print(\"   - These cases should be reviewed to identify recurring failure patterns such as lexical overlap or structural variation.\")\n",
    "\n",
    "print(\"\\n6. THRESHOLD RECOMMENDATIONS\")\n",
    "print(f\"   - Balanced F1 threshold: {threshold_balanced:.3f}\")\n",
    "print(f\"   - High-precision threshold: {threshold_high_prec_text}\")\n",
    "print(f\"   - High-recall threshold: {threshold_high_recall_text}\")\n",
    "print(\"   - Choose the operating point based on whether false alarms or missed plagiarism is more costly.\")\n",
    "\n",
    "print(\"\\n7. OVERALL CONCLUSION\")\n",
    "print(\"   - The notebook now evaluates the full fusion pipeline on the real MSRP train/test split.\")\n",
    "print(\"   - ROC comparison shows whether the fusion model dominates each individual method across thresholds, not just at one cutoff.\")\n",
    "print(\"   - Together with the F1 comparison, this gives clearer evidence that combining semantic and syntactic signals is more effective than using any single method alone.\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 100)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
