{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "953fd65a",
   "metadata": {},
   "source": [
    "# 3rd Phase: Fusion model\n",
    "Combining Syntactic (Tree Edit Distance) and Semantic (BERT) branches into a single classification\n",
    "\n",
    "The goal is to train a logistical regression model using features from both branches to predict the liklihood of plagiarism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1c45d83192facfc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 29 test pairs for fusion analysis\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, roc_curve\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import pickle\n",
    "\n",
    "# Load models\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "model_bert = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "test_pairs = [\n",
    "## STANDARD TEST PAIRS (from previous notebooks)\n",
    "    # Direct copies and near-copies\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat.\"),           # Exact copy\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat\"),            # No punctuation\n",
    "    (\"The cat sat on the mat.\", \"The  cat  sat  on  the  mat.\"),      # Extra spaces\n",
    "    \n",
    "    # Paraphrases with same meaning\n",
    "    (\"The cat sat on the mat.\", \"On the mat, the cat was sitting.\"),  # Structural change\n",
    "    (\"The cat sat on the mat.\", \"The feline rested on the rug.\"),     # Synonym replacement\n",
    "    (\"The quick brown fox jumps.\", \"A fast brown fox leaps.\"),        # Partial synonym\n",
    "    \n",
    "    # Different sentences\n",
    "    (\"The cat sat on the mat.\", \"The dog ran in the park.\"),          # Different content\n",
    "    (\"I love programming.\", \"She enjoys reading books.\"),             # Completely different\n",
    "    (\"The weather is nice today.\", \"It's raining outside.\"),          # Opposite meaning\n",
    "    \n",
    "    # Edge cases\n",
    "    (\"Short.\", \"Short.\"),                                             # Very short\n",
    "    (\"A B C D E F G\", \"A B C D E F G\"),                              # Repeated words\n",
    "    (\"\", \"\"),                                                         # Empty strings\n",
    "## ADDITIONAL TEST PAIRS (for semantic evaluation)\n",
    "    # Polysemy (word sense ambiguity) - Critical for BERT vs SpaCy\n",
    "    (\"He went to the bank to deposit money.\", \"He went to the river bank to fish.\"),           # Bank: financial vs. riverbank\n",
    "    (\"I saw the bat in the cave.\", \"The baseball bat is broken.\"),                             # Bat: animal vs. sports equipment\n",
    "    (\"The light is very bright.\", \"Can you carry this light package?\"),                        # Light: brightness vs. weight\n",
    "    \n",
    "    # Synonymy (different words, same meaning) - Tests semantic understanding\n",
    "    (\"The car is fast.\", \"The automobile is quick.\"),                                          # car/automobile, fast/quick\n",
    "    (\"He commenced the project.\", \"He started the work.\"),                                     # commenced/started, project/work\n",
    "    (\"The substantial building is ornate.\", \"The large building is decorated.\"),               # substantial/large, ornate/decorated\n",
    "    \n",
    "    # Negation (opposite meaning) - Tests if methods catch semantic inversion\n",
    "    (\"The weather is good.\", \"The weather is bad.\"),                                           # Direct negation\n",
    "    (\"The solution is simple.\", \"The problem is complex.\"),                                    # Opposite adjectives\n",
    "    (\"I like this movie.\", \"I dislike this movie.\"),                                           # Explicit negation\n",
    "    \n",
    "    # Semantic similarity without word overlap - Pure meaning test\n",
    "    (\"A man walks down the street.\", \"A person strolls along the road.\"),                      # Very few shared words, similar meaning\n",
    "    (\"The student studied the textbook.\", \"The pupil learned from the book.\"),                 # Different words, same semantic content\n",
    "    \n",
    "    # Partial overlap (high word similarity, low semantic similarity)\n",
    "    (\"The bank is on the river.\", \"The bank account is overdrawn.\"),                           # \"bank\" is only shared word; different meanings\n",
    "    (\"I read the book last night.\", \"The book was read by many people.\"),                      # Same key words (book, read) but different focus/meaning\n",
    "    \n",
    "    # Length variation (tests robustness to sentence length)\n",
    "    (\"Cat sat.\", \"The cat sat on the mat for hours and hours.\"),                               # Very different lengths, similar core meaning\n",
    "    (\"Go.\", \"You should go to the store and buy milk immediately.\"),                           # Minimal vs. detailed\n",
    "    \n",
    "    # Metaphor/Figurative language\n",
    "    (\"Time is money.\", \"Time has value.\"),                                                     # Metaphor vs. literal\n",
    "    (\"The world is a stage.\", \"Life is like a theater performance.\"),                          # Figurative expressions\n",
    "]\n",
    "\n",
    "print(f\"Loaded {len(test_pairs)} test pairs for fusion analysis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea6f478e",
   "metadata": {},
   "source": [
    "## Method 1: Tree Edit Distance (Syntactic)\n",
    "Calculates minimum edit operations to convert one dependency tree to another"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9665682bd5a7951e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TED Similarity: 0.857\n",
      "(Higher score = more similar dependency structures)\n"
     ]
    }
   ],
   "source": [
    "def get_dependency_tree(text):\n",
    "    \"\"\"Extract dependency tree structure from sentence.\"\"\"\n",
    "    doc = nlp(text)\n",
    "    tree = {}\n",
    "    for token in doc:\n",
    "        tree[token.i] = {\n",
    "            'text': token.text,\n",
    "            'pos': token.pos_,\n",
    "            'dep': token.dep_,\n",
    "            'head': token.head.i\n",
    "        }\n",
    "    return tree\n",
    "\n",
    "def tree_edit_distance(tree1, tree2):\n",
    "    \"\"\"\n",
    "    Simple Tree Edit Distance approximation.\n",
    "    Compares dependency structures without full algorithm (O(n^3) complexity).\n",
    "    \"\"\"\n",
    "    if not tree1 and not tree2:\n",
    "        return 0.0\n",
    "    if not tree1 or not tree2:\n",
    "        return 1.0\n",
    "    \n",
    "    # Extract dependency edges\n",
    "    edges1 = set()\n",
    "    edges2 = set()\n",
    "    \n",
    "    for i, node1 in tree1.items():\n",
    "        edges1.add((node1['dep'], node1['pos']))\n",
    "    \n",
    "    for i, node2 in tree2.items():\n",
    "        edges2.add((node2['dep'], node2['pos']))\n",
    "    \n",
    "    # Jaccard similarity on edges\n",
    "    if not edges1 and not edges2:\n",
    "        return 1.0\n",
    "    \n",
    "    intersection = len(edges1.intersection(edges2))\n",
    "    union = len(edges1.union(edges2))\n",
    "    \n",
    "    return intersection / union if union > 0 else 0.0\n",
    "\n",
    "def sentence_similarity_ted(sent1, sent2):\n",
    "    \"\"\"Compute similarity using Tree Edit Distance.\"\"\"\n",
    "    tree1 = get_dependency_tree(sent1)\n",
    "    tree2 = get_dependency_tree(sent2)\n",
    "    return tree_edit_distance(tree1, tree2)\n",
    "\n",
    "# Demo\n",
    "s1 = \"The cat sat on the mat.\"\n",
    "s2 = \"On the mat, the cat was sitting.\"\n",
    "\n",
    "print(f\"TED Similarity: {sentence_similarity_ted(s1, s2):.3f}\")\n",
    "print(\"(Higher score = more similar dependency structures)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83262ade",
   "metadata": {},
   "source": [
    "## Method 2: Baseline Similarity (notebook 2)\n",
    "Jaccard similarity as a quick baseline for structural matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "91282393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jaccard Similarity: 0.375\n"
     ]
    }
   ],
   "source": [
    "def sentence_similarity_jaccard(sent1, sent2):\n",
    "    \"\"\"Jaccard similarity baseline.\"\"\"\n",
    "    words1 = set(sent1.lower().split())\n",
    "    words2 = set(sent2.lower().split())\n",
    "    \n",
    "    if not words1 and not words2:\n",
    "        return 1.0\n",
    "    if not words1 or not words2:\n",
    "        return 0.0\n",
    "    \n",
    "    intersection = len(words1.intersection(words2))\n",
    "    union = len(words1.union(words2))\n",
    "    return intersection / union\n",
    "\n",
    "# Demo\n",
    "print(f\"Jaccard Similarity: {sentence_similarity_jaccard(s1, s2):.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30db037d",
   "metadata": {},
   "source": [
    "## Feature Extraction\n",
    "Extract all relevent features for each pair"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c9f502e",
   "metadata": {},
   "source": [
    "#### Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "54916598",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Extraction Demo:\n",
      "============================================================\n",
      "bert           : 0.930\n",
      "ted            : 0.857\n",
      "jaccard        : 0.375\n",
      "len_ratio      : 0.719\n"
     ]
    }
   ],
   "source": [
    "def extract_features(sent1, sent2):\n",
    "    \"\"\"Extract semantic and syntactic features for a sentence pair.\"\"\"\n",
    "    \n",
    "    # Semantic features\n",
    "    embeddings = model_bert.encode([sent1, sent2])\n",
    "    bert_similarity = np.dot(embeddings[0], embeddings[1]) / (\n",
    "        np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1])\n",
    "    )\n",
    "    \n",
    "    # Syntactic features\n",
    "    ted_similarity = sentence_similarity_ted(sent1, sent2)\n",
    "    \n",
    "    # Baseline features\n",
    "    jaccard_similarity = sentence_similarity_jaccard(sent1, sent2)\n",
    "    \n",
    "    # Length ratio (robustness feature)\n",
    "    len_ratio = min(len(sent1), len(sent2)) / max(len(sent1), len(sent2)) if max(len(sent1), len(sent2)) > 0 else 0.0\n",
    "    \n",
    "    return {\n",
    "        'bert': bert_similarity,\n",
    "        'ted': ted_similarity,\n",
    "        'jaccard': jaccard_similarity,\n",
    "        'len_ratio': len_ratio\n",
    "    }\n",
    "\n",
    "# Demo\n",
    "print(\"Feature Extraction Demo:\")\n",
    "print(\"=\" * 60)\n",
    "features = extract_features(s1, s2)\n",
    "for feature_name, value in features.items():\n",
    "    print(f\"{feature_name:<15}: {value:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26a1aa86",
   "metadata": {},
   "source": [
    "#### Building feature map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8a13c18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features for all pairs...\n",
      "  Processed 10/29 pairs...\n",
      "  Processed 20/29 pairs...\n",
      "\n",
      "Feature Matrix Shape: (29, 7)\n",
      "\n",
      "Feature Summary Statistics:\n",
      "================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bert</th>\n",
       "      <th>ted</th>\n",
       "      <th>jaccard</th>\n",
       "      <th>len_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>29.000</td>\n",
       "      <td>29.000</td>\n",
       "      <td>29.000</td>\n",
       "      <td>29.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.671</td>\n",
       "      <td>0.671</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.271</td>\n",
       "      <td>0.338</td>\n",
       "      <td>0.328</td>\n",
       "      <td>0.266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.067</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.506</td>\n",
       "      <td>0.364</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.652</td>\n",
       "      <td>0.800</td>\n",
       "      <td>0.333</td>\n",
       "      <td>0.857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.930</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         bert     ted  jaccard  len_ratio\n",
       "count  29.000  29.000   29.000     29.000\n",
       "mean    0.671   0.671    0.405      0.786\n",
       "std     0.271   0.338    0.328      0.266\n",
       "min     0.067   0.000    0.000      0.000\n",
       "25%     0.506   0.364    0.200      0.760\n",
       "50%     0.652   0.800    0.333      0.857\n",
       "75%     0.930   1.000    0.600      0.957\n",
       "max     1.000   1.000    1.000      1.000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Extracting features for all pairs...\")\n",
    "feature_list = []\n",
    "\n",
    "for i, (sent1, sent2) in enumerate(test_pairs):\n",
    "    features = extract_features(sent1, sent2)\n",
    "    features['pair_id'] = i\n",
    "    features['sentence_1'] = sent1\n",
    "    features['sentence_2'] = sent2\n",
    "    feature_list.append(features)\n",
    "    \n",
    "    if (i + 1) % 10 == 0:\n",
    "        print(f\"  Processed {i + 1}/{len(test_pairs)} pairs...\")\n",
    "\n",
    "feature_df = pd.DataFrame(feature_list)\n",
    "\n",
    "print(f\"\\nFeature Matrix Shape: {feature_df.shape}\")\n",
    "print(\"\\nFeature Summary Statistics:\")\n",
    "print(\"=\" * 80)\n",
    "display(feature_df[['bert', 'ted', 'jaccard', 'len_ratio']].describe().round(3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15d1f95a",
   "metadata": {},
   "source": [
    "#### Correlation and Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "292e3d91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55084 characters]",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Feature Correlation Insights:\n",
      "================================================================================\n",
      "               bert       ted   jaccard  len_ratio\n",
      "bert       1.000000  0.257085  0.770717   0.157109\n",
      "ted        0.257085  1.000000  0.337151   0.531403\n",
      "jaccard    0.770717  0.337151  1.000000   0.149126\n",
      "len_ratio  0.157109  0.531403  0.149126   1.000000\n"
     ]
    }
   ],
   "source": [
    "# Correlation heatmap\n",
    "plt.figure(figsize=(8, 6))\n",
    "feature_corr = feature_df[['bert', 'ted', 'jaccard', 'len_ratio']].corr()\n",
    "sns.heatmap(feature_corr, annot=True, cmap='coolwarm', center=0, vmin=-1, vmax=1, \n",
    "            square=True, fmt='.3f', cbar_kws={'label': 'Correlation'})\n",
    "plt.title('Feature Correlation Matrix\\n(Low correlation = good for fusion model)', fontsize=12, fontweight='bold')\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\nFeature Correlation Insights:\")\n",
    "print(\"=\" * 80)\n",
    "print(feature_corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49482f8f",
   "metadata": {},
   "source": [
    "## Ground Truth Labels\n",
    "As a demo within the notebook labels will be assigned based on semantic similarity threshold\n",
    "(Real training would use MSRP corpus labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "4bea7544",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label Distribution:\n",
      "================================================================================\n",
      "Paraphrases (1): 14\n",
      "Non-Paraphrases (0): 15\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 34312 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# For demo purposes: pairs with BERT similarity > 0.7 are \"paraphrases\" (label=1)\n",
    "feature_df['label'] = (feature_df['bert'] > 0.7).astype(int)\n",
    "\n",
    "print(\"Label Distribution:\")\n",
    "print(\"=\" * 80)\n",
    "label_counts = feature_df['label'].value_counts()\n",
    "print(f\"Paraphrases (1): {label_counts.get(1, 0)}\")\n",
    "print(f\"Non-Paraphrases (0): {label_counts.get(0, 0)}\")\n",
    "\n",
    "# Visualize label distribution\n",
    "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
    "\n",
    "# Count plot\n",
    "label_counts.plot(kind='bar', ax=axes[0], color=['#d62728', '#2ca02c'])\n",
    "axes[0].set_title('Label Distribution')\n",
    "axes[0].set_xlabel('Label (0=Different, 1=Paraphrase)')\n",
    "axes[0].set_ylabel('Count')\n",
    "axes[0].set_xticklabels(['Different', 'Paraphrase'], rotation=0)\n",
    "\n",
    "# Feature distributions by label\n",
    "feature_df.boxplot(column='bert', by='label', ax=axes[1])\n",
    "axes[1].set_title('BERT Similarity by Label')\n",
    "axes[1].set_xlabel('Label')\n",
    "axes[1].set_ylabel('BERT Similarity')\n",
    "plt.suptitle('')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64c8b61b",
   "metadata": {},
   "source": [
    "## Model Training & Evaluation\n",
    "Train Logistical Regression on merged features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0d0a4596",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: 23\n",
      "Test set size: 6\n",
      "\n",
      "Model Training Complete!\n",
      "================================================================================\n",
      "Training Accuracy: 1.000\n",
      "Test Accuracy: 0.833\n",
      "\n",
      "Feature Importance (Model Coefficients):\n",
      "================================================================================\n",
      "bert           : +1.7731\n",
      "ted            : +0.2136\n",
      "jaccard        : +0.9931\n",
      "len_ratio      : +0.0666\n"
     ]
    }
   ],
   "source": [
    "# Prepare features\n",
    "X = feature_df[['bert', 'ted', 'jaccard', 'len_ratio']].values\n",
    "y = feature_df['label'].values\n",
    "\n",
    "# Standardize features (important for Logistic Regression)\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "\n",
    "# Train-test split (80-20)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n",
    "\n",
    "print(f\"Training set size: {len(X_train)}\")\n",
    "print(f\"Test set size: {len(X_test)}\")\n",
    "\n",
    "# Train Logistic Regression\n",
    "model = LogisticRegression(random_state=42, max_iter=1000)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "print(\"\\nModel Training Complete!\")\n",
    "print(\"=\" * 80)\n",
    "print(f\"Training Accuracy: {model.score(X_train, y_train):.3f}\")\n",
    "print(f\"Test Accuracy: {model.score(X_test, y_test):.3f}\")\n",
    "\n",
    "# Feature importance (coefficients)\n",
    "print(\"\\nFeature Importance (Model Coefficients):\")\n",
    "print(\"=\" * 80)\n",
    "feature_names = ['bert', 'ted', 'jaccard', 'len_ratio']\n",
    "for name, coef in zip(feature_names, model.coef_[0]):\n",
    "    print(f\"{name:<15}: {coef:+.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ce47f9c",
   "metadata": {},
   "source": [
    "#### Classification report & Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ed9f0973",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:\n",
      "================================================================================\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   Different       0.75      1.00      0.86         3\n",
      "  Paraphrase       1.00      0.67      0.80         3\n",
      "\n",
      "    accuracy                           0.83         6\n",
      "   macro avg       0.88      0.83      0.83         6\n",
      "weighted avg       0.88      0.83      0.83         6\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56032 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Predictions\n",
    "y_pred = model.predict(X_test)\n",
    "y_pred_proba = model.predict_proba(X_test)[:, 1]\n",
    "\n",
    "print(\"Classification Report:\")\n",
    "print(\"=\" * 80)\n",
    "print(classification_report(y_test, y_pred, target_names=['Different', 'Paraphrase']))\n",
    "\n",
    "# Confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
    "\n",
    "# Confusion matrix heatmap\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0], cbar=False)\n",
    "axes[0].set_title('Confusion Matrix')\n",
    "axes[0].set_xlabel('Predicted')\n",
    "axes[0].set_ylabel('Actual')\n",
    "axes[0].set_xticklabels(['Different', 'Paraphrase'])\n",
    "axes[0].set_yticklabels(['Different', 'Paraphrase'])\n",
    "\n",
    "# ROC Curve\n",
    "fpr, tpr, _ = roc_curve(y_test, y_pred_proba)\n",
    "auc = roc_auc_score(y_test, y_pred_proba)\n",
    "axes[1].plot(fpr, tpr, label=f'ROC Curve (AUC = {auc:.3f})', linewidth=2)\n",
    "axes[1].plot([0, 1], [0, 1], 'k--', label='Random Classifier')\n",
    "axes[1].set_xlabel('False Positive Rate')\n",
    "axes[1].set_ylabel('True Positive Rate')\n",
    "axes[1].set_title('ROC Curve')\n",
    "axes[1].legend()\n",
    "axes[1].grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bc9f488",
   "metadata": {},
   "source": [
    "#### Visualize feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6dc8a0c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 34336 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Importance Summary:\n",
      "================================================================================\n",
      "BERT (Semantic)          : +1.7731 ↑ Increases paraphrase likelihood\n",
      "TED (Syntactic)          : +0.2136 ↑ Increases paraphrase likelihood\n",
      "Jaccard (Baseline)       : +0.9931 ↑ Increases paraphrase likelihood\n",
      "Length Ratio             : +0.0666 ↑ Increases paraphrase likelihood\n"
     ]
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "feature_names = ['BERT (Semantic)', 'TED (Syntactic)', 'Jaccard (Baseline)', 'Length Ratio']\n",
    "coefficients = model.coef_[0]\n",
    "colors = ['green' if c > 0 else 'red' for c in coefficients]\n",
    "\n",
    "plt.barh(feature_names, coefficients, color=colors, alpha=0.7)\n",
    "plt.xlabel('Model Coefficient (Impact on Prediction)')\n",
    "plt.title('Feature Importance in Fusion Model')\n",
    "plt.axvline(x=0, color='black', linestyle='--', linewidth=0.8)\n",
    "plt.grid(axis='x', alpha=0.3)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"Feature Importance Summary:\")\n",
    "print(\"=\" * 80)\n",
    "for name, coef in zip(feature_names, coefficients):\n",
    "    direction = \"↑ Increases paraphrase likelihood\" if coef > 0 else \"↓ Decreases paraphrase likelihood\"\n",
    "    print(f\"{name:<25}: {coef:+.4f} {direction}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45112ea7",
   "metadata": {},
   "source": [
    "## Ablation Study\n",
    "Comparing performance when removing individual features, evaluate importance of each"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "611b74f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ablation Study Results:\n",
      "================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Full (All Features)</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Without bert</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Without ted</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Without jaccard</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Without len_ratio</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model  F1-Score  Precision  Recall\n",
       "0  Full (All Features)       0.8        1.0   0.667\n",
       "1         Without bert       0.4        0.5   0.333\n",
       "2          Without ted       0.8        1.0   0.667\n",
       "3      Without jaccard       1.0        1.0   1.000\n",
       "4    Without len_ratio       0.8        1.0   0.667"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 50060 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score\n",
    "\n",
    "ablation_results = []\n",
    "\n",
    "# Full model\n",
    "full_pred = model.predict(X_test)\n",
    "full_score = f1_score(y_test, full_pred)\n",
    "ablation_results.append({\n",
    "    'Model': 'Full (All Features)',\n",
    "    'F1-Score': full_score,\n",
    "    'Precision': precision_score(y_test, full_pred),\n",
    "    'Recall': recall_score(y_test, full_pred)\n",
    "})\n",
    "\n",
    "# Remove each feature one at a time\n",
    "feature_idx_map = {'bert': 0, 'ted': 1, 'jaccard': 2, 'len_ratio': 3}\n",
    "\n",
    "for feature_name, idx in feature_idx_map.items():\n",
    "    # Create feature set without this feature\n",
    "    mask = np.ones(X_scaled.shape[1], dtype=bool)\n",
    "    mask[idx] = False\n",
    "    X_ablated = X_scaled[:, mask]\n",
    "    \n",
    "    # Train and test\n",
    "    X_train_abl, X_test_abl, _, _ = train_test_split(X_ablated, y, test_size=0.2, random_state=42)\n",
    "    model_abl = LogisticRegression(random_state=42, max_iter=1000)\n",
    "    model_abl.fit(X_train_abl, y_train)\n",
    "    \n",
    "    y_pred_abl = model_abl.predict(X_test_abl)\n",
    "    \n",
    "    ablation_results.append({\n",
    "        'Model': f'Without {feature_name}',\n",
    "        'F1-Score': f1_score(y_test, y_pred_abl),\n",
    "        'Precision': precision_score(y_test, y_pred_abl),\n",
    "        'Recall': recall_score(y_test, y_pred_abl)\n",
    "    })\n",
    "\n",
    "ablation_df = pd.DataFrame(ablation_results)\n",
    "\n",
    "print(\"Ablation Study Results:\")\n",
    "print(\"=\" * 80)\n",
    "display(ablation_df.round(3))\n",
    "\n",
    "# Visualization\n",
    "fig, ax = plt.subplots(figsize=(10, 5))\n",
    "ablation_df.set_index('Model')[['F1-Score', 'Precision', 'Recall']].plot(kind='bar', ax=ax, alpha=0.8)\n",
    "ax.set_title('Model Performance: Ablation Study')\n",
    "ax.set_ylabel('Score')\n",
    "ax.set_xlabel('Model Configuration')\n",
    "ax.set_ylim(0, 1)\n",
    "ax.legend(loc='lower left')\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.grid(axis='y', alpha=0.3)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
