{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "953fd65a",
   "metadata": {},
   "source": [
    "# 3rd Phase: Fusion model\n",
    "Combining Syntactic (Tree Edit Distance) and Semantic (BERT) branches into a single classification\n",
    "\n",
    "The goal is to train a logistical regression model using features from both branches to predict the liklihood of plagiarism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1c45d83192facfc6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0d416371b6c438bbcdc750301240d68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7b1657fc75346a18c7d2a7a3f096313",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config_sentence_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae3f3f6247a5440abfe73a13fa69283e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "README.md: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "629b99694e654a469d04c4b42385ba69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "sentence_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec714af14453479f84ea91fbdeb0a5e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/612 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70e163142ab04a53ae2b6d1eb83e08b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "model.safetensors:   0%|          | 0.00/90.9M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "538150e747e04bccb34bb691ebb1f3ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/350 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad4b584258754059b6f04052a8605813",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "vocab.txt: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cef8aab72d51486dba9737d8e30e4239",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer.json: 0.00B [00:00, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "979d08190f344bcaa4c87b3d697dc702",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "049f02fb6f964b148a613c65cf0a3a69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 29 test pairs for fusion analysis\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import classification_report, confusion_matrix, roc_auc_score, roc_curve\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "from scipy.stats import mannwhitneyu\n",
    "from statsmodels.stats.multitest import multipletests\n",
    "\n",
    "# import pickle\n",
    "\n",
    "# Load models\n",
    "nlp = spacy.load(\"en_core_web_lg\")\n",
    "model_bert = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",
    "\n",
    "test_pairs = [\n",
    "## STANDARD TEST PAIRS (from previous notebooks)\n",
    "    # Direct copies and near-copies\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat.\"),           # Exact copy\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat\"),            # No punctuation\n",
    "    (\"The cat sat on the mat.\", \"The  cat  sat  on  the  mat.\"),      # Extra spaces\n",
    "    \n",
    "    # Paraphrases with same meaning\n",
    "    (\"The cat sat on the mat.\", \"On the mat, the cat was sitting.\"),  # Structural change\n",
    "    (\"The cat sat on the mat.\", \"The feline rested on the rug.\"),     # Synonym replacement\n",
    "    (\"The quick brown fox jumps.\", \"A fast brown fox leaps.\"),        # Partial synonym\n",
    "    \n",
    "    # Different sentences\n",
    "    (\"The cat sat on the mat.\", \"The dog ran in the park.\"),          # Different content\n",
    "    (\"I love programming.\", \"She enjoys reading books.\"),             # Completely different\n",
    "    (\"The weather is nice today.\", \"It's raining outside.\"),          # Opposite meaning\n",
    "    \n",
    "    # Edge cases\n",
    "    (\"Short.\", \"Short.\"),                                             # Very short\n",
    "    (\"A B C D E F G\", \"A B C D E F G\"),                               # Repeated words\n",
    "    (\"\", \"\"),                                                         # Empty strings\n",
    "    \n",
    "## ADDITIONAL TEST PAIRS (for semantic evaluation)\n",
    "    # Polysemy (word sense ambiguity) - Critical for BERT vs SpaCy\n",
    "    (\"He went to the bank to deposit money.\", \"He went to the river bank to fish.\"),           # Bank: financial vs. riverbank\n",
    "    (\"I saw the bat in the cave.\", \"The baseball bat is broken.\"),                             # Bat: animal vs. sports equipment\n",
    "    (\"The light is very bright.\", \"Can you carry this light package?\"),                        # Light: brightness vs. weight\n",
    "    \n",
    "    # Synonymy (different words, same meaning) - Tests semantic understanding\n",
    "    (\"The car is fast.\", \"The automobile is quick.\"),                                          # car/automobile, fast/quick\n",
    "    (\"He commenced the project.\", \"He started the work.\"),                                     # commenced/started, project/work\n",
    "    (\"The substantial building is ornate.\", \"The large building is decorated.\"),               # substantial/large, ornate/decorated\n",
    "    \n",
    "    # Negation (opposite meaning) - Tests if methods catch semantic inversion\n",
    "    (\"The weather is good.\", \"The weather is bad.\"),                                           # Direct negation\n",
    "    (\"The solution is simple.\", \"The problem is complex.\"),                                    # Opposite adjectives\n",
    "    (\"I like this movie.\", \"I dislike this movie.\"),                                           # Explicit negation\n",
    "    \n",
    "    # Semantic similarity without word overlap - Pure meaning test\n",
    "    (\"A man walks down the street.\", \"A person strolls along the road.\"),                      # Very few shared words, similar meaning\n",
    "    (\"The student studied the textbook.\", \"The pupil learned from the book.\"),                 # Different words, same semantic content\n",
    "    \n",
    "    # Partial overlap (high word similarity, low semantic similarity)\n",
    "    (\"The bank is on the river.\", \"The bank account is overdrawn.\"),                           # \"bank\" is only shared word; different meanings\n",
    "    (\"I read the book last night.\", \"The book was read by many people.\"),                      # Same key words (book, read) but different focus\n",
    "    \n",
    "    # Length variation (tests robustness to sentence length)\n",
    "    (\"Cat sat.\", \"The cat sat on the mat for hours and hours.\"),                               # Very different lengths, similar core meaning\n",
    "    (\"Go.\", \"You should go to the store and buy milk immediately.\"),                           # Minimal vs. detailed\n",
    "    \n",
    "    # Metaphor/Figurative language\n",
    "    (\"Time is money.\", \"Time has value.\"),                                                     # Metaphor vs. literal\n",
    "    (\"The world is a stage.\", \"Life is like a theater performance.\"),                          # Figurative expressions\n",
    "]\n",
    "\n",
    "print(f\"Loaded {len(test_pairs)} test pairs for fusion analysis\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea6f478e",
   "metadata": {},
   "source": [
    "## Method 1: Tree Edit Distance (Syntactic)\n",
    "Calculates minimum edit operations to convert one dependency tree to another"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9665682bd5a7951e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tree Structure Demo:\n",
      "======================================================================\n",
      "Sentence: 'The cat sat on the mat.'\n",
      "Root token ID: 2\n",
      "\n",
      "Tree nodes:\n",
      "  ID 0: 'The       ' POS=DET    DEP=det      Children=[]\n",
      "  ID 1: 'cat       ' POS=NOUN   DEP=nsubj    Children=[0]\n",
      "  ID 2: 'sat       ' POS=VERB   DEP=ROOT     Children=[1, 3, 6]\n",
      "  ID 3: 'on        ' POS=ADP    DEP=prep     Children=[5]\n",
      "  ID 4: 'the       ' POS=DET    DEP=det      Children=[]\n",
      "  ID 5: 'mat       ' POS=NOUN   DEP=pobj     Children=[4]\n",
      "  ID 6: '.         ' POS=PUNCT  DEP=punct    Children=[]\n"
     ]
    }
   ],
   "source": [
    "def build_tree_from_dependencies(text):\n",
    "    \"\"\"Build hierarchical tree structure from dependency parse.\"\"\"\n",
    "    doc = nlp(text)\n",
    "    \n",
    "    # node dictionary with parent-child relationships\n",
    "    tree = {}\n",
    "    root_token = None\n",
    "    \n",
    "    for token in doc:\n",
    "        tree[token.i] = {\n",
    "            'text': token.text,\n",
    "            'lemma': token.lemma_,\n",
    "            'pos': token.pos_,\n",
    "            'dep': token.dep_,\n",
    "            'head_id': token.head.i,\n",
    "            'children': []\n",
    "        }\n",
    "    \n",
    "    # parent-child relationships\n",
    "    for token in doc:\n",
    "        if token.head.i != token.i:  # Not root\n",
    "            tree[token.head.i]['children'].append(token.i)\n",
    "        else:  # Root node\n",
    "            root_token = token.i\n",
    "    \n",
    "    return tree, root_token\n",
    "\n",
    "# small demo\n",
    "s1 = \"The cat sat on the mat.\"\n",
    "tree1, root1 = build_tree_from_dependencies(s1)\n",
    "\n",
    "print(\"Tree Structure Demo:\")\n",
    "print(\"=\" * 70)\n",
    "print(f\"Sentence: '{s1}'\")\n",
    "print(f\"Root token ID: {root1}\")\n",
    "print(\"\\nTree nodes:\")\n",
    "for node_id in sorted(tree1.keys()):\n",
    "    node = tree1[node_id]\n",
    "    print(f\"  ID {node_id}: '{node['text']:<10}' POS={node['pos']:<6} DEP={node['dep']:<8} Children={node['children']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d36fd1dd",
   "metadata": {},
   "source": [
    "### Helper functions for tree comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d936f69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Postorder Traversals:\n",
      "======================================================================\n",
      "Sentence 1: 'The cat sat on the mat.'\n",
      "Postorder: [0, 1, 4, 5, 3, 6, 2]\n",
      "Labels: [('DET', 'det'), ('NOUN', 'nsubj'), ('DET', 'det'), ('NOUN', 'pobj'), ('ADP', 'prep'), ('PUNCT', 'punct'), ('VERB', 'ROOT')]\n",
      "\n",
      "Sentence 2: 'On the mat, the cat was sitting.'\n",
      "Postorder: [1, 2, 0, 3, 4, 5, 6, 8, 7]\n",
      "Labels: [('DET', 'det'), ('NOUN', 'pobj'), ('ADP', 'prep'), ('PUNCT', 'punct'), ('DET', 'det'), ('NOUN', 'nsubj'), ('AUX', 'aux'), ('PUNCT', 'punct'), ('VERB', 'ROOT')]\n"
     ]
    }
   ],
   "source": [
    "def get_nodes_postorder(node_id, tree):\n",
    "    \"\"\"\n",
    "    Get all nodes in postorder traversal (children before parents).\n",
    "    This is required for Zhang-Shasha algorithm.\n",
    "    \"\"\"\n",
    "    nodes = []\n",
    "    if node_id in tree:\n",
    "        # traverse children first\n",
    "        for child_id in tree[node_id]['children']:\n",
    "            nodes.extend(get_nodes_postorder(child_id, tree))\n",
    "        # then add parent\n",
    "        nodes.append(node_id)\n",
    "    return nodes\n",
    "\n",
    "def node_label(node_id, tree):\n",
    "    \"\"\"\n",
    "    Get label for a node (combination of POS tag and dependency relation).\n",
    "    Two nodes match if they have the same (POS, DEP) pair.\n",
    "    \"\"\"\n",
    "    if node_id in tree:\n",
    "        return (tree[node_id]['pos'], tree[node_id]['dep'])\n",
    "    return ('', '')\n",
    "\n",
    "def nodes_match(node_id1, tree1, node_id2, tree2):\n",
    "    \"\"\"Check if two nodes have matching labels (same POS and DEP).\"\"\"\n",
    "    label1 = node_label(node_id1, tree1)\n",
    "    label2 = node_label(node_id2, tree2)\n",
    "    return label1 == label2\n",
    "\n",
    "# small demo\n",
    "s2 = \"On the mat, the cat was sitting.\"\n",
    "tree2, root2 = build_tree_from_dependencies(s2)\n",
    "\n",
    "print(\"Postorder Traversals:\")\n",
    "print(\"=\" * 70)\n",
    "print(f\"Sentence 1: '{s1}'\")\n",
    "nodes1 = get_nodes_postorder(root1, tree1)\n",
    "print(f\"Postorder: {nodes1}\")\n",
    "labels1 = [node_label(n, tree1) for n in nodes1]\n",
    "print(f\"Labels: {labels1}\")\n",
    "\n",
    "print(f\"\\nSentence 2: '{s2}'\")\n",
    "nodes2 = get_nodes_postorder(root2, tree2)\n",
    "print(f\"Postorder: {nodes2}\")\n",
    "labels2 = [node_label(n, tree2) for n in nodes2]\n",
    "print(f\"Labels: {labels2}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f99f2ea",
   "metadata": {},
   "source": [
    "### Zhang-Shasha Tree Edit Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6a91abd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tree Edit Distance Calculation:\n",
      "======================================================================\n",
      "Sentence 1: 'The cat sat on the mat.'\n",
      "Sentence 2: 'On the mat, the cat was sitting.'\n",
      "\n",
      "Tree 1 size: 7 nodes\n",
      "Tree 2 size: 9 nodes\n",
      "\n",
      "Zhang-Shasha Tree Edit Distance Similarity: 0.444\n",
      "(Higher = more similar tree structures)\n"
     ]
    }
   ],
   "source": [
    "def tree_edit_distance_zhang_shasha(tree1, tree2, root1, root2):\n",
    "    \"\"\"\n",
    "    Full Tree Edit Distance using Zhang-Shasha algorithm.\n",
    "    \n",
    "    Compares two trees by:\n",
    "    1. Computing minimum edit operations (insert, delete, replace)\n",
    "    2. Preserving hierarchical structure and node ordering\n",
    "    3. Matching nodes by (POS tag, dependency relation)\n",
    "    \n",
    "    Returns: Normalized similarity score (0-1)\n",
    "    \"\"\"\n",
    "    \n",
    "    # get postorder traversals of both trees\n",
    "    nodes1 = get_nodes_postorder(root1, tree1)\n",
    "    nodes2 = get_nodes_postorder(root2, tree2)\n",
    "    \n",
    "    m, n = len(nodes1), len(nodes2)\n",
    "    \n",
    "    # Initialize DP table\n",
    "    # dp[i][j] = minimum edit distance between first i nodes of tree1 and first j nodes of tree2\n",
    "    dp = [[0] * (n + 1) for _ in range(m + 1)]\n",
    "    \n",
    "    # Base cases: converting empty tree to/from another tree\n",
    "    for i in range(m + 1):\n",
    "        dp[i][0] = i  # Delete all i nodes from tree1\n",
    "    for j in range(n + 1):\n",
    "        dp[0][j] = j  # Insert all j nodes to get tree2\n",
    "    \n",
    "    # Fill DP table\n",
    "    for i in range(1, m + 1):\n",
    "        for j in range(1, n + 1):\n",
    "            node1 = nodes1[i - 1]\n",
    "            node2 = nodes2[j - 1]\n",
    "            \n",
    "            if nodes_match(node1, tree1, node2, tree2):\n",
    "                # nodes match: no operation needed\n",
    "                cost = dp[i - 1][j - 1]\n",
    "            else:\n",
    "                # nodes don't match: choose cheapest operation\n",
    "                delete_cost = dp[i - 1][j] + 1      # Delete node from tree1\n",
    "                insert_cost = dp[i][j - 1] + 1      # Insert node into tree1\n",
    "                replace_cost = dp[i - 1][j - 1] + 1 # Replace node\n",
    "                cost = min(delete_cost, insert_cost, replace_cost)\n",
    "            \n",
    "            dp[i][j] = cost\n",
    "    \n",
    "    # convert edit distance to similarity (0-1)\n",
    "    max_size = max(m, n)\n",
    "    if max_size == 0:\n",
    "        return 1.0\n",
    "    \n",
    "    edit_distance = dp[m][n]\n",
    "    similarity = 1.0 - (edit_distance / max_size)\n",
    "    \n",
    "    return max(0.0, similarity)\n",
    "\n",
    "# small demo\n",
    "ted_score = tree_edit_distance_zhang_shasha(tree1, tree2, root1, root2)\n",
    "\n",
    "print(\"Tree Edit Distance Calculation:\")\n",
    "print(\"=\" * 70)\n",
    "print(f\"Sentence 1: '{s1}'\")\n",
    "print(f\"Sentence 2: '{s2}'\")\n",
    "print(f\"\\nTree 1 size: {len(tree1)} nodes\")\n",
    "print(f\"Tree 2 size: {len(tree2)} nodes\")\n",
    "print(f\"\\nZhang-Shasha Tree Edit Distance Similarity: {ted_score:.3f}\")\n",
    "print(\"(Higher = more similar tree structures)\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c01a28b5",
   "metadata": {},
   "source": [
    "### Wrapper for TED similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "16436c70",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sentence Similarity using Tree Edit Distance:\n",
      "======================================================================\n",
      "\n",
      "'The cat sat on the mat.'\n",
      "  vs\n",
      "'The cat sat on the mat.'\n",
      "  → Similarity: 1.000\n",
      "\n",
      "'The cat sat on the mat.'\n",
      "  vs\n",
      "'On the mat, the cat was sitting.'\n",
      "  → Similarity: 0.444\n",
      "\n",
      "'The cat sat on the mat.'\n",
      "  vs\n",
      "'The dog ran in the park.'\n",
      "  → Similarity: 1.000\n"
     ]
    }
   ],
   "source": [
    "def sentence_similarity_ted(sent1, sent2):\n",
    "    \"\"\"\n",
    "    Compute similarity between two sentences using Tree Edit Distance.\n",
    "    Analyzes full hierarchical tree structure.\n",
    "    \"\"\"\n",
    "    if not sent1.strip() or not sent2.strip():\n",
    "        return 0.0\n",
    "    \n",
    "    try:\n",
    "        tree1, root1 = build_tree_from_dependencies(sent1)\n",
    "        tree2, root2 = build_tree_from_dependencies(sent2)\n",
    "        return tree_edit_distance_zhang_shasha(tree1, tree2, root1, root2)\n",
    "    except:\n",
    "        return 0.0\n",
    "\n",
    "# Demo\n",
    "print(\"\\nSentence Similarity using Tree Edit Distance:\")\n",
    "print(\"=\" * 70)\n",
    "test_pairs_demo = [\n",
    "    (\"The cat sat on the mat.\", \"The cat sat on the mat.\"),\n",
    "    (\"The cat sat on the mat.\", \"On the mat, the cat was sitting.\"),\n",
    "    (\"The cat sat on the mat.\", \"The dog ran in the park.\"),\n",
    "]\n",
    "\n",
    "for sent1, sent2 in test_pairs_demo:\n",
    "    sim = sentence_similarity_ted(sent1, sent2)\n",
    "    print(f\"\\n'{sent1}'\")\n",
    "    print(f\"  vs\")\n",
    "    print(f\"'{sent2}'\")\n",
    "    print(f\"  → Similarity: {sim:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83262ade",
   "metadata": {},
   "source": [
    "## Method 2: Baseline Similarity (notebook 2)\n",
    "Jaccard similarity as a quick baseline for structural matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "91282393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jaccard Similarity: 0.375\n"
     ]
    }
   ],
   "source": [
    "def sentence_similarity_jaccard(sent1, sent2):\n",
    "    \"\"\"Jaccard similarity baseline.\"\"\"\n",
    "    words1 = set(sent1.lower().split())\n",
    "    words2 = set(sent2.lower().split())\n",
    "    \n",
    "    if not words1 and not words2:\n",
    "        return 1.0\n",
    "    if not words1 or not words2:\n",
    "        return 0.0\n",
    "    \n",
    "    intersection = len(words1.intersection(words2))\n",
    "    union = len(words1.union(words2))\n",
    "    return intersection / union\n",
    "\n",
    "# Demo\n",
    "print(f\"Jaccard Similarity: {sentence_similarity_jaccard(s1, s2):.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30db037d",
   "metadata": {},
   "source": [
    "## Feature Extraction\n",
    "Extract all relevent features for each pair"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c9f502e",
   "metadata": {},
   "source": [
    "#### Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "54916598",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Extraction Demo:\n",
      "============================================================\n",
      "bert           : 0.930\n",
      "ted            : 0.444\n",
      "jaccard        : 0.375\n",
      "len_ratio      : 0.719\n"
     ]
    }
   ],
   "source": [
    "def extract_features(sent1, sent2):\n",
    "    \"\"\"Extract semantic and syntactic features for a sentence pair.\"\"\"\n",
    "    \n",
    "    # Semantic features\n",
    "    embeddings = model_bert.encode([sent1, sent2])\n",
    "    bert_similarity = np.dot(embeddings[0], embeddings[1]) / (\n",
    "        np.linalg.norm(embeddings[0]) * np.linalg.norm(embeddings[1])\n",
    "    )\n",
    "    \n",
    "    # Syntactic features\n",
    "    ted_similarity = sentence_similarity_ted(sent1, sent2)\n",
    "    \n",
    "    # Baseline features\n",
    "    jaccard_similarity = sentence_similarity_jaccard(sent1, sent2)\n",
    "    \n",
    "    # Length ratio (robustness feature)\n",
    "    len_ratio = min(len(sent1), len(sent2)) / max(len(sent1), len(sent2)) if max(len(sent1), len(sent2)) > 0 else 0.0\n",
    "    \n",
    "    return {\n",
    "        'bert': bert_similarity,\n",
    "        'ted': ted_similarity,\n",
    "        'jaccard': jaccard_similarity,\n",
    "        'len_ratio': len_ratio\n",
    "    }\n",
    "\n",
    "# Demo\n",
    "print(\"Feature Extraction Demo:\")\n",
    "print(\"=\" * 60)\n",
    "features = extract_features(s1, s2)\n",
    "for feature_name, value in features.items():\n",
    "    print(f\"{feature_name:<15}: {value:.3f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26a1aa86",
   "metadata": {},
   "source": [
    "#### Building feature map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e8a13c18",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features for all pairs...\n",
      "  Processed 10/29 pairs...\n",
      "  Processed 20/29 pairs...\n",
      "\n",
      "Feature Matrix Shape: (29, 7)\n",
      "\n",
      "Feature Summary Statistics:\n",
      "================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bert</th>\n",
       "      <th>ted</th>\n",
       "      <th>jaccard</th>\n",
       "      <th>len_ratio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>29.000</td>\n",
       "      <td>29.000</td>\n",
       "      <td>29.000</td>\n",
       "      <td>29.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.671</td>\n",
       "      <td>0.663</td>\n",
       "      <td>0.405</td>\n",
       "      <td>0.786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.271</td>\n",
       "      <td>0.330</td>\n",
       "      <td>0.328</td>\n",
       "      <td>0.266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.067</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.506</td>\n",
       "      <td>0.429</td>\n",
       "      <td>0.200</td>\n",
       "      <td>0.760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.652</td>\n",
       "      <td>0.714</td>\n",
       "      <td>0.333</td>\n",
       "      <td>0.857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.930</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.600</td>\n",
       "      <td>0.957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         bert     ted  jaccard  len_ratio\n",
       "count  29.000  29.000   29.000     29.000\n",
       "mean    0.671   0.663    0.405      0.786\n",
       "std     0.271   0.330    0.328      0.266\n",
       "min     0.067   0.000    0.000      0.000\n",
       "25%     0.506   0.429    0.200      0.760\n",
       "50%     0.652   0.714    0.333      0.857\n",
       "75%     0.930   1.000    0.600      0.957\n",
       "max     1.000   1.000    1.000      1.000"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"Extracting features for all pairs...\")\n",
    "feature_list = []\n",
    "\n",
    "for i, (sent1, sent2) in enumerate(test_pairs):\n",
    "    features = extract_features(sent1, sent2)\n",
    "    features['pair_id'] = i\n",
    "    features['sentence_1'] = sent1\n",
    "    features['sentence_2'] = sent2\n",
    "    feature_list.append(features)\n",
    "    \n",
    "    if (i + 1) % 10 == 0:\n",
    "        print(f\"  Processed {i + 1}/{len(test_pairs)} pairs...\")\n",
    "\n",
    "feature_df = pd.DataFrame(feature_list)\n",
    "\n",
    "print(f\"\\nFeature Matrix Shape: {feature_df.shape}\")\n",
    "print(\"\\nFeature Summary Statistics:\")\n",
    "print(\"=\" * 80)\n",
    "display(feature_df[['bert', 'ted', 'jaccard', 'len_ratio']].describe().round(3))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "15d1f95a",
   "metadata": {},
   "source": [
    "#### Correlation and Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "292e3d91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 54916 characters]",
      "text/plain": [
       "<Figure size 800x600 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Feature Correlation Insights:\n",
      "================================================================================\n",
      "               bert       ted   jaccard  len_ratio\n",
      "bert       1.000000  0.219195  0.770717   0.157109\n",
      "ted        0.219195  1.000000  0.272899   0.621469\n",
      "jaccard    0.770717  0.272899  1.000000   0.149126\n",
      "len_ratio  0.157109  0.621469  0.149126   1.000000\n"
     ]
    }
   ],
   "source": [
    "# Correlation heatmap\n",
    "plt.figure(figsize=(8, 6))\n",
    "feature_corr = feature_df[['bert', 'ted', 'jaccard', 'len_ratio']].corr()\n",
    "sns.heatmap(feature_corr, annot=True, cmap='coolwarm', center=0, vmin=-1, vmax=1, \n",
    "            square=True, fmt='.3f', cbar_kws={'label': 'Correlation'})\n",
    "plt.title('Feature Correlation Matrix\\n(Low correlation = good for fusion model)', fontsize=12, fontweight='bold')\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\nFeature Correlation Insights:\")\n",
    "print(\"=\" * 80)\n",
    "print(feature_corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49482f8f",
   "metadata": {},
   "source": [
    "## Ground Truth Labels\n",
    "As a demo within the notebook labels will be assigned based on semantic similarity threshold\n",
    "(Real training would use MSRP corpus labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4bea7544",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label Distribution:\n",
      "================================================================================\n",
      "Paraphrases (1): 14\n",
      "Non-Paraphrases (0): 15\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 34312 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# For demo purposes: pairs with BERT similarity > 0.7 are \"paraphrases\" (label=1)\n",
    "feature_df['label'] = (feature_df['bert'] > 0.7).astype(int)\n",
    "\n",
    "print(\"Label Distribution:\")\n",
    "print(\"=\" * 80)\n",
    "label_counts = feature_df['label'].value_counts()\n",
    "print(f\"Paraphrases (1): {label_counts.get(1, 0)}\")\n",
    "print(f\"Non-Paraphrases (0): {label_counts.get(0, 0)}\")\n",
    "\n",
    "# Visualize label distribution\n",
    "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
    "\n",
    "# Count plot\n",
    "label_counts.plot(kind='bar', ax=axes[0], color=['#d62728', '#2ca02c'])\n",
    "axes[0].set_title('Label Distribution')\n",
    "axes[0].set_xlabel('Label (0=Different, 1=Paraphrase)')\n",
    "axes[0].set_ylabel('Count')\n",
    "axes[0].set_xticklabels(['Different', 'Paraphrase'], rotation=0)\n",
    "\n",
    "# Feature distributions by label\n",
    "feature_df.boxplot(column='bert', by='label', ax=axes[1])\n",
    "axes[1].set_title('BERT Similarity by Label')\n",
    "axes[1].set_xlabel('Label')\n",
    "axes[1].set_ylabel('BERT Similarity')\n",
    "plt.suptitle('')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db7ede30",
   "metadata": {},
   "source": [
    "### Statistical Feature Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "27a74280",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mann-Whitney U Test: Feature Discrimination Power\n",
      "==========================================================================================\n",
      "Tests whether each feature separates Paraphrases from Non-Paraphrases\n",
      "\n",
      "Feature         U-stat     p-value    p-corrected  Effect   Sig?   Med(Para)  Med(Diff)\n",
      "------------------------------------------------------------------------------------------\n",
      "bert            210.0      0.0000     0.0000       -1.000 (LARGE) ✓✓✓    0.938      0.506\n",
      "jaccard         189.5      0.0002     0.0005       -0.805 (LARGE) ✓✓✓    0.600      0.200\n",
      "ted             142.0      0.1013     0.1351       -0.352 (MED  ) ✗      0.929      0.429\n",
      "len_ratio       125.5      0.3821     0.3821       -0.195 (SMALL) ✗      0.899      0.818\n",
      "\n",
      "==========================================================================================\n",
      "Interpretation:\n",
      "  - Effect size: >0 means feature is higher for paraphrases\n",
      "  - p-corrected: Benjamini-Hochberg FDR correction (controls false discoveries)\n",
      "  - ✓✓✓ = p<0.001 (highly significant), ✓✓ = p<0.01, ✓ = p<0.05, ✗ = not significant\n"
     ]
    }
   ],
   "source": [
    "def mann_whitney_feature_test(df, feature_col, label_col='label'):\n",
    "    \"\"\"\n",
    "    Test if feature distributions differ significantly between classes.\n",
    "    \n",
    "    Returns:\n",
    "    - U statistic, p-value, effect size, medians\n",
    "    \"\"\"\n",
    "    group0 = df[df[label_col] == 0][feature_col].dropna().values\n",
    "    group1 = df[df[label_col] == 1][feature_col].dropna().values\n",
    "    \n",
    "    if len(group0) < 2 or len(group1) < 2:\n",
    "        return None  # Not enough data\n",
    "    \n",
    "    # Two-sided test: are distributions different?\n",
    "    u_stat, p_val = mannwhitneyu(group1, group0, alternative='two-sided')\n",
    "    \n",
    "    # Rank-biserial correlation (effect size: -1 to +1)\n",
    "    n1, n0 = len(group1), len(group0)\n",
    "    rbc = 1 - (2 * u_stat) / (n1 * n0)\n",
    "    \n",
    "    return {\n",
    "        \"feature\": feature_col,\n",
    "        \"n_paraphrase\": n1,\n",
    "        \"n_different\": n0,\n",
    "        \"U_statistic\": u_stat,\n",
    "        \"p_value\": p_val,\n",
    "        \"effect_size\": rbc,  # >0.5 = large, >0.3 = medium, >0.1 = small\n",
    "        \"median_paraphrase\": np.median(group1),\n",
    "        \"median_different\": np.median(group0),\n",
    "    }\n",
    "\n",
    "# Test all features\n",
    "print(\"Mann-Whitney U Test: Feature Discrimination Power\")\n",
    "print(\"=\" * 90)\n",
    "print(\"Tests whether each feature separates Paraphrases from Non-Paraphrases\\n\")\n",
    "\n",
    "test_results = []\n",
    "for feature in ['bert', 'ted', 'jaccard', 'len_ratio']:\n",
    "    result = mann_whitney_feature_test(feature_df, feature)\n",
    "    if result:\n",
    "        test_results.append(result)\n",
    "\n",
    "# Apply Benjamini-Hochberg correction for multiple testing\n",
    "p_values = [r['p_value'] for r in test_results]\n",
    "reject, p_corrected, _, _ = multipletests(p_values, alpha=0.05, method='fdr_bh')\n",
    "\n",
    "# Add corrected p-values\n",
    "for i, result in enumerate(test_results):\n",
    "    result['p_corrected'] = p_corrected[i]\n",
    "    result['significant'] = reject[i]\n",
    "\n",
    "# Display results\n",
    "results_df = pd.DataFrame(test_results)\n",
    "results_df = results_df.sort_values('effect_size', ascending=False, key=abs)\n",
    "\n",
    "print(f\"{'Feature':<15} {'U-stat':<10} {'p-value':<10} {'p-corrected':<12} {'Effect':<8} {'Sig?':<6} {'Med(Para)':<10} {'Med(Diff)'}\")\n",
    "print(\"-\" * 90)\n",
    "\n",
    "for _, row in results_df.iterrows():\n",
    "    sig_mark = \"✓✓✓\" if row['p_corrected'] < 0.001 else (\"✓✓\" if row['p_corrected'] < 0.01 else (\"✓\" if row['significant'] else \"✗\"))\n",
    "    effect_interp = \"LARGE\" if abs(row['effect_size']) > 0.5 else (\"MED\" if abs(row['effect_size']) > 0.3 else \"SMALL\")\n",
    "    \n",
    "    print(f\"{row['feature']:<15} {row['U_statistic']:<10.1f} {row['p_value']:<10.4f} {row['p_corrected']:<12.4f} \"\n",
    "          f\"{row['effect_size']:>+.3f} ({effect_interp:<5}) {sig_mark:<6} {row['median_paraphrase']:<10.3f} {row['median_different']:.3f}\")\n",
    "\n",
    "print(\"\\n\" + \"=\" * 90)\n",
    "print(\"Interpretation:\")\n",
    "print(\"  - Effect size: >0 means feature is higher for paraphrases\")\n",
    "print(\"  - p-corrected: Benjamini-Hochberg FDR correction (controls false discoveries)\")\n",
    "print(\"  - ✓✓✓ = p<0.001 (highly significant), ✓✓ = p<0.01, ✓ = p<0.05, ✗ = not significant\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d9fc526",
   "metadata": {},
   "source": [
    "#### Visualize Mann-Whitney U"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2c877a29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 87020 characters]",
      "text/plain": [
       "<Figure size 1400x500 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Key Insight: BERT has the strongest separation between classes\n",
      "Effect size: -1.000 (p=0.0000)\n"
     ]
    }
   ],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
    "\n",
    "# 1: Effect sizes\n",
    "features_sorted = results_df.sort_values('effect_size')\n",
    "colors = ['green' if x else 'red' for x in features_sorted['significant']]\n",
    "\n",
    "axes[0].barh(features_sorted['feature'], features_sorted['effect_size'], color=colors, alpha=0.7)\n",
    "axes[0].axvline(x=0, color='black', linestyle='--', linewidth=1)\n",
    "axes[0].axvline(x=0.3, color='orange', linestyle=':', linewidth=1, alpha=0.5, label='Medium effect')\n",
    "axes[0].axvline(x=-0.3, color='orange', linestyle=':', linewidth=1, alpha=0.5)\n",
    "axes[0].set_xlabel('Effect Size (Rank-Biserial Correlation)')\n",
    "axes[0].set_title('Feature Discrimination: Effect Sizes\\n(Green = significant after FDR correction)')\n",
    "axes[0].legend()\n",
    "axes[0].grid(axis='x', alpha=0.3)\n",
    "\n",
    "# 2: Distribution comparison for best feature\n",
    "best_feature = results_df.iloc[0]['feature']\n",
    "group0 = feature_df[feature_df['label'] == 0][best_feature]\n",
    "group1 = feature_df[feature_df['label'] == 1][best_feature]\n",
    "\n",
    "axes[1].hist(group0, bins=10, alpha=0.6, label=f'Non-Paraphrase (n={len(group0)})', color='red', edgecolor='black')\n",
    "axes[1].hist(group1, bins=10, alpha=0.6, label=f'Paraphrase (n={len(group1)})', color='green', edgecolor='black')\n",
    "axes[1].axvline(np.median(group0), color='darkred', linestyle='--', linewidth=2, label=f'Median Non-Para: {np.median(group0):.3f}')\n",
    "axes[1].axvline(np.median(group1), color='darkgreen', linestyle='--', linewidth=2, label=f'Median Para: {np.median(group1):.3f}')\n",
    "axes[1].set_xlabel(f'{best_feature.upper()} Score')\n",
    "axes[1].set_ylabel('Frequency')\n",
    "axes[1].set_title(f'Distribution Comparison: {best_feature.upper()}\\n(Best discriminating feature)')\n",
    "axes[1].legend()\n",
    "axes[1].grid(axis='y', alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(f\"\\nKey Insight: {best_feature.upper()} has the strongest separation between classes\")\n",
    "print(f\"Effect size: {results_df.iloc[0]['effect_size']:+.3f} (p={results_df.iloc[0]['p_corrected']:.4f})\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64c8b61b",
   "metadata": {},
   "source": [
    "## Model Training & Evaluation\n",
    "Train Logistical Regression on merged features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0d0a4596",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set size: 23\n",
      "Test set size: 6\n",
      "\n",
      "Model Training Complete!\n",
      "================================================================================\n",
      "Training Accuracy: 1.000\n",
      "Test Accuracy: 0.833\n",
      "\n",
      "Feature Importance (Model Coefficients):\n",
      "================================================================================\n",
      "bert           : +1.7775\n",
      "ted            : +0.2371\n",
      "jaccard        : +0.9941\n",
      "len_ratio      : +0.0367\n"
     ]
    }
   ],
   "source": [
    "# Prepare features\n",
    "X = feature_df[['bert', 'ted', 'jaccard', 'len_ratio']].values\n",
    "y = feature_df['label'].values\n",
    "\n",
    "# Standardize features (important for Logistic Regression)\n",
    "scaler = StandardScaler()\n",
    "X_scaled = scaler.fit_transform(X)\n",
    "\n",
    "# Train-test split (80-20)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.2, random_state=42)\n",
    "\n",
    "print(f\"Training set size: {len(X_train)}\")\n",
    "print(f\"Test set size: {len(X_test)}\")\n",
    "\n",
    "# Train Logistic Regression\n",
    "model = LogisticRegression(random_state=42, max_iter=1000)\n",
    "model.fit(X_train, y_train)\n",
    "\n",
    "print(\"\\nModel Training Complete!\")\n",
    "print(\"=\" * 80)\n",
    "print(f\"Training Accuracy: {model.score(X_train, y_train):.3f}\")\n",
    "print(f\"Test Accuracy: {model.score(X_test, y_test):.3f}\")\n",
    "\n",
    "# Feature importance (coefficients)\n",
    "print(\"\\nFeature Importance (Model Coefficients):\")\n",
    "print(\"=\" * 80)\n",
    "feature_names = ['bert', 'ted', 'jaccard', 'len_ratio']\n",
    "for name, coef in zip(feature_names, model.coef_[0]):\n",
    "    print(f\"{name:<15}: {coef:+.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ce47f9c",
   "metadata": {},
   "source": [
    "#### Classification report & Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ed9f0973",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification Report:\n",
      "================================================================================\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "   Different       0.75      1.00      0.86         3\n",
      "  Paraphrase       1.00      0.67      0.80         3\n",
      "\n",
      "    accuracy                           0.83         6\n",
      "   macro avg       0.88      0.83      0.83         6\n",
      "weighted avg       0.88      0.83      0.83         6\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56032 characters]",
      "text/plain": [
       "<Figure size 1200x400 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Predictions\n",
    "y_pred = model.predict(X_test)\n",
    "y_pred_proba = model.predict_proba(X_test)[:, 1]\n",
    "\n",
    "print(\"Classification Report:\")\n",
    "print(\"=\" * 80)\n",
    "print(classification_report(y_test, y_pred, target_names=['Different', 'Paraphrase']))\n",
    "\n",
    "# Confusion matrix\n",
    "cm = confusion_matrix(y_test, y_pred)\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
    "\n",
    "# Confusion matrix heatmap\n",
    "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues', ax=axes[0], cbar=False)\n",
    "axes[0].set_title('Confusion Matrix')\n",
    "axes[0].set_xlabel('Predicted')\n",
    "axes[0].set_ylabel('Actual')\n",
    "axes[0].set_xticklabels(['Different', 'Paraphrase'])\n",
    "axes[0].set_yticklabels(['Different', 'Paraphrase'])\n",
    "\n",
    "# ROC Curve\n",
    "fpr, tpr, _ = roc_curve(y_test, y_pred_proba)\n",
    "auc_score = roc_auc_score(y_test, y_pred_proba)\n",
    "axes[1].plot(fpr, tpr, label=f'ROC Curve (AUC = {auc_score:.3f})', linewidth=2)\n",
    "axes[1].plot([0, 1], [0, 1], 'k--', label='Random Classifier')\n",
    "axes[1].set_xlabel('False Positive Rate')\n",
    "axes[1].set_ylabel('True Positive Rate')\n",
    "axes[1].set_title('ROC Curve')\n",
    "axes[1].legend()\n",
    "axes[1].grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bc9f488",
   "metadata": {},
   "source": [
    "#### Visualize feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "6dc8a0c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 34332 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Importance Summary:\n",
      "================================================================================\n",
      "BERT (Semantic)          : +1.7775 ↑ Increases paraphrase likelihood\n",
      "TED (Syntactic)          : +0.2371 ↑ Increases paraphrase likelihood\n",
      "Jaccard (Baseline)       : +0.9941 ↑ Increases paraphrase likelihood\n",
      "Length Ratio             : +0.0367 ↑ Increases paraphrase likelihood\n"
     ]
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 5))\n",
    "feature_names = ['BERT (Semantic)', 'TED (Syntactic)', 'Jaccard (Baseline)', 'Length Ratio']\n",
    "coefficients = model.coef_[0]\n",
    "colors = ['green' if c > 0 else 'red' for c in coefficients]\n",
    "\n",
    "plt.barh(feature_names, coefficients, color=colors, alpha=0.7)\n",
    "plt.xlabel('Model Coefficient (Impact on Prediction)')\n",
    "plt.title('Feature Importance in Fusion Model')\n",
    "plt.axvline(x=0, color='black', linestyle='--', linewidth=0.8)\n",
    "plt.grid(axis='x', alpha=0.3)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"Feature Importance Summary:\")\n",
    "print(\"=\" * 80)\n",
    "for name, coef in zip(feature_names, coefficients):\n",
    "    direction = \"↑ Increases paraphrase likelihood\" if coef > 0 else \"↓ Decreases paraphrase likelihood\"\n",
    "    print(f\"{name:<25}: {coef:+.4f} {direction}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45112ea7",
   "metadata": {},
   "source": [
    "## Ablation Study\n",
    "Comparing performance when removing individual features, evaluate importance of each"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "611b74f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ablation Study Results:\n",
      "================================================================================\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Model</th>\n",
       "      <th>F1-Score</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Full (All Features)</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Without bert</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Without ted</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Without jaccard</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Without len_ratio</td>\n",
       "      <td>0.8</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 Model  F1-Score  Precision  Recall\n",
       "0  Full (All Features)       0.8        1.0   0.667\n",
       "1         Without bert       0.4        0.5   0.333\n",
       "2          Without ted       0.8        1.0   0.667\n",
       "3      Without jaccard       1.0        1.0   1.000\n",
       "4    Without len_ratio       0.8        1.0   0.667"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 50060 characters]",
      "text/plain": [
       "<Figure size 1000x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score, precision_score, recall_score\n",
    "\n",
    "ablation_results = []\n",
    "\n",
    "# Full model\n",
    "full_pred = model.predict(X_test)\n",
    "full_score = f1_score(y_test, full_pred)\n",
    "ablation_results.append({\n",
    "    'Model': 'Full (All Features)',\n",
    "    'F1-Score': full_score,\n",
    "    'Precision': precision_score(y_test, full_pred),\n",
    "    'Recall': recall_score(y_test, full_pred)\n",
    "})\n",
    "\n",
    "# Remove each feature one at a time\n",
    "feature_idx_map = {'bert': 0, 'ted': 1, 'jaccard': 2, 'len_ratio': 3}\n",
    "\n",
    "for feature_name, idx in feature_idx_map.items():\n",
    "    # Create feature set without this feature\n",
    "    mask = np.ones(X_scaled.shape[1], dtype=bool)\n",
    "    mask[idx] = False\n",
    "    X_ablated = X_scaled[:, mask]\n",
    "    \n",
    "    # Train and test\n",
    "    X_train_abl, X_test_abl, _, _ = train_test_split(X_ablated, y, test_size=0.2, random_state=42)\n",
    "    model_abl = LogisticRegression(random_state=42, max_iter=1000)\n",
    "    model_abl.fit(X_train_abl, y_train)\n",
    "    \n",
    "    y_pred_abl = model_abl.predict(X_test_abl)\n",
    "    \n",
    "    ablation_results.append({\n",
    "        'Model': f'Without {feature_name}',\n",
    "        'F1-Score': f1_score(y_test, y_pred_abl),\n",
    "        'Precision': precision_score(y_test, y_pred_abl),\n",
    "        'Recall': recall_score(y_test, y_pred_abl)\n",
    "    })\n",
    "\n",
    "ablation_df = pd.DataFrame(ablation_results)\n",
    "\n",
    "print(\"Ablation Study Results:\")\n",
    "print(\"=\" * 80)\n",
    "display(ablation_df.round(3))\n",
    "\n",
    "# Visualization\n",
    "fig, ax = plt.subplots(figsize=(10, 5))\n",
    "ablation_df.set_index('Model')[['F1-Score', 'Precision', 'Recall']].plot(kind='bar', ax=ax, alpha=0.8)\n",
    "ax.set_title('Model Performance: Ablation Study')\n",
    "ax.set_ylabel('Score')\n",
    "ax.set_xlabel('Model Configuration')\n",
    "ax.set_ylim(0, 1)\n",
    "ax.legend(loc='lower left')\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.grid(axis='y', alpha=0.3)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7949424",
   "metadata": {},
   "source": [
    "#### Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "53e53eb5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FUSION MODEL ANALYSIS SUMMARY\n",
      "================================================================================\n",
      "\n",
      "1. Feature Correlations:\n",
      "   - BERT & TED correlation: 0.219\n",
      "     → Low = both branches provide independent information ✓\n",
      "\n",
      "2. Feature Importance:\n",
      "   - Most important feature: BERT (Semantic)\n",
      "     → Coefficient: 1.7775\n",
      "\n",
      "3. Model Performance:\n",
      "   - Test Accuracy: 0.833\n",
      "   - Test F1-Score: 0.800\n",
      "   - ROC AUC: 0.889\n",
      "\n",
      "4. Ablation Insights:\n",
      "   - Max performance drop when removing feature: 0.400\n",
      "   - All features contribute to model performance ✓\n",
      "\n",
      "5. Recommendations for Next Phase:\n",
      "   - Train on full MSRP corpus (35 pairs too small)\n",
      "   - Experiment with SVM or Random Forest for non-linear relationships\n",
      "   - Optimize threshold for precision vs. recall trade-off\n"
     ]
    }
   ],
   "source": [
    "print(\"FUSION MODEL ANALYSIS SUMMARY\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "print(\"\\n1. Feature Correlations:\")\n",
    "print(f\"   - BERT & TED correlation: {feature_corr.loc['bert', 'ted']:.3f}\")\n",
    "print(f\"     → Low = both branches provide independent information ✓\")\n",
    "\n",
    "print(\"\\n2. Feature Importance:\")\n",
    "most_important = feature_names[np.argmax(np.abs(model.coef_[0]))]\n",
    "print(f\"   - Most important feature: {most_important}\")\n",
    "print(f\"     → Coefficient: {np.max(np.abs(model.coef_[0])):.4f}\")\n",
    "\n",
    "print(\"\\n3. Model Performance:\")\n",
    "print(f\"   - Test Accuracy: {model.score(X_test, y_test):.3f}\")\n",
    "print(f\"   - Test F1-Score: {f1_score(y_test, y_pred):.3f}\")\n",
    "print(f\"   - ROC AUC: {auc_score:.3f}\")\n",
    "\n",
    "print(\"\\n4. Ablation Insights:\")\n",
    "performance_drop = ablation_df.iloc[0]['F1-Score'] - ablation_df['F1-Score'].iloc[1:].min()\n",
    "print(f\"   - Max performance drop when removing feature: {performance_drop:.3f}\")\n",
    "print(f\"   - All features contribute to model performance ✓\")\n",
    "\n",
    "print(\"\\n5. Recommendations for Next Phase:\")\n",
    "print(\"   - Train on full MSRP corpus (35 pairs too small)\")\n",
    "print(\"   - Experiment with SVM or Random Forest for non-linear relationships\")\n",
    "print(\"   - Optimize threshold for precision vs. recall trade-off\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
