{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "dd72d1539056a64",
   "metadata": {},
   "source": [
    "Import Cell\n"
   ]
  },
  {
   "cell_type": "code",
   "id": "12579bf734bb1a92",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:41.942156089Z",
     "start_time": "2025-12-14T01:12:31.060344929Z"
    }
   },
   "source": [
    "import spacy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from collections import Counter\n",
    "import string\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "nlp_lg = spacy.load(\"en_core_web_lg\")\n",
    "nlp_trf = spacy.load(\"en_core_web_trf\")\n",
    "\n",
    "nlp = nlp_lg\n",
    "\n",
    "# Test sentences covering different similarity types\n",
    "test_sentences = [\n",
    "    # Direct copies and near-copies\n",
    "    \"The cat sat on the mat.\",\n",
    "    \"The cat sat on the mat\",\n",
    "    \"The  cat  sat  on  the  mat.\",\n",
    "    \n",
    "    # Paraphrases\n",
    "    \"On the mat, the cat was sitting.\",\n",
    "    \"The feline rested on the rug.\",\n",
    "    \n",
    "    # Structural changes\n",
    "    \"The quick brown fox jumps over the lazy dog.\",\n",
    "    \"Over the lazy dog jumps the quick brown fox.\",\n",
    "    \n",
    "    # Different content\n",
    "    \"The dog ran in the park.\",\n",
    "    \"I love programming.\",\n",
    "    \"She enjoys reading books.\",\n",
    "    \n",
    "    # Edge cases\n",
    "    \"Short.\",\n",
    "    \"A B C D E F G\",\n",
    "    \"\"\n",
    "]\n",
    "\n",
    "print(f\"Loaded {len(test_sentences)} test sentences\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded 13 test sentences\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "markdown",
   "id": "1c26616777253f10",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "id": "e003ac06a58cfbb4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:42.630828162Z",
     "start_time": "2025-12-14T01:12:41.953456434Z"
    }
   },
   "source": [
    "def analyze_text_statistics(sentences):\n",
    "    stats = []\n",
    "    for i, sent in enumerate(sentences):\n",
    "        doc = nlp(sent)\n",
    "        words = [token.text for token in doc if not token.is_space]\n",
    "        alpha_words = [token.text for token in doc if token.is_alpha]\n",
    "        \n",
    "        stats.append({\n",
    "            'sentence_id': i,\n",
    "            'sentence': sent,\n",
    "            'char_length': len(sent),\n",
    "            'word_count': len(words),\n",
    "            'alpha_word_count': len(alpha_words),\n",
    "            'avg_word_length': np.mean([len(word) for word in alpha_words]) if alpha_words else 0,\n",
    "            'has_punctuation': any(token.is_punct for token in doc),\n",
    "            'is_empty': len(sent.strip()) == 0\n",
    "        })\n",
    "    \n",
    "    return pd.DataFrame(stats)\n",
    "\n",
    "stats_df = analyze_text_statistics(test_sentences)\n",
    "display(stats_df)\n",
    "\n",
    "# Visualize basic statistics\n",
    "fig, axes = plt.subplots(2, 2, figsize=(12, 8))\n",
    "\n",
    "# Character length distribution\n",
    "axes[0,0].bar(range(len(stats_df)), stats_df['char_length'])\n",
    "axes[0,0].set_title('Character Length by Sentence')\n",
    "axes[0,0].set_xlabel('Sentence ID')\n",
    "axes[0,0].set_ylabel('Characters')\n",
    "\n",
    "# Word count distribution\n",
    "axes[0,1].bar(range(len(stats_df)), stats_df['word_count'])\n",
    "axes[0,1].set_title('Word Count by Sentence')\n",
    "axes[0,1].set_xlabel('Sentence ID')\n",
    "axes[0,1].set_ylabel('Words')\n",
    "\n",
    "# Average word length\n",
    "axes[1,0].bar(range(len(stats_df)), stats_df['avg_word_length'])\n",
    "axes[1,0].set_title('Average Word Length')\n",
    "axes[1,0].set_xlabel('Sentence ID')\n",
    "axes[1,0].set_ylabel('Characters')\n",
    "\n",
    "# Sentence type breakdown\n",
    "types = ['With Punctuation' if x else 'No Punctuation' for x in stats_df['has_punctuation']]\n",
    "type_counts = pd.Series(types).value_counts()\n",
    "axes[1,1].pie(type_counts.values, labels=type_counts.index, autopct='%1.1f%%')\n",
    "axes[1,1].set_title('Punctuation Distribution')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "    sentence_id                                      sentence  char_length  \\\n",
       "0             0                       The cat sat on the mat.           23   \n",
       "1             1                        The cat sat on the mat           22   \n",
       "2             2                  The  cat  sat  on  the  mat.           28   \n",
       "3             3              On the mat, the cat was sitting.           32   \n",
       "4             4                 The feline rested on the rug.           29   \n",
       "5             5  The quick brown fox jumps over the lazy dog.           44   \n",
       "6             6  Over the lazy dog jumps the quick brown fox.           44   \n",
       "7             7                      The dog ran in the park.           24   \n",
       "8             8                           I love programming.           19   \n",
       "9             9                     She enjoys reading books.           25   \n",
       "10           10                                        Short.            6   \n",
       "11           11                                 A B C D E F G           13   \n",
       "12           12                                                          0   \n",
       "\n",
       "    word_count  alpha_word_count  avg_word_length  has_punctuation  is_empty  \n",
       "0            7                 6         2.833333             True     False  \n",
       "1            6                 6         2.833333            False     False  \n",
       "2            7                 6         2.833333             True     False  \n",
       "3            9                 7         3.428571             True     False  \n",
       "4            7                 6         3.833333             True     False  \n",
       "5           10                 9         3.888889             True     False  \n",
       "6           10                 9         3.888889             True     False  \n",
       "7            7                 6         3.000000             True     False  \n",
       "8            4                 3         5.333333             True     False  \n",
       "9            5                 4         5.250000             True     False  \n",
       "10           2                 1         5.000000             True     False  \n",
       "11           7                 7         1.000000            False     False  \n",
       "12           0                 0         0.000000            False      True  "
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence_id</th>\n",
       "      <th>sentence</th>\n",
       "      <th>char_length</th>\n",
       "      <th>word_count</th>\n",
       "      <th>alpha_word_count</th>\n",
       "      <th>avg_word_length</th>\n",
       "      <th>has_punctuation</th>\n",
       "      <th>is_empty</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>The cat sat on the mat.</td>\n",
       "      <td>23</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>2.833333</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>The cat sat on the mat</td>\n",
       "      <td>22</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>2.833333</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>The  cat  sat  on  the  mat.</td>\n",
       "      <td>28</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>2.833333</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>On the mat, the cat was sitting.</td>\n",
       "      <td>32</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>3.428571</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>The feline rested on the rug.</td>\n",
       "      <td>29</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>3.833333</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>The quick brown fox jumps over the lazy dog.</td>\n",
       "      <td>44</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>3.888889</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>Over the lazy dog jumps the quick brown fox.</td>\n",
       "      <td>44</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>3.888889</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>The dog ran in the park.</td>\n",
       "      <td>24</td>\n",
       "      <td>7</td>\n",
       "      <td>6</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>I love programming.</td>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>5.333333</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>She enjoys reading books.</td>\n",
       "      <td>25</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5.250000</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>Short.</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>A B C D E F G</td>\n",
       "      <td>13</td>\n",
       "      <td>7</td>\n",
       "      <td>7</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td></td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1200x800 with 4 Axes>"
      ],
      "image/png": "[encoded data, 79388 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "5e488a878a5cfccb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:44.526513959Z",
     "start_time": "2025-12-14T01:12:42.694215942Z"
    }
   },
   "source": [
    "\n",
    "class TextPreprocessor:\n",
    "    def __init__(self):\n",
    "        self.nlp = spacy.load(\"en_core_web_lg\")\n",
    "    \n",
    "    def extract_linguistic_features(self, text):\n",
    "        \"\"\" Extract linguistic features \"\"\"\n",
    "        doc = self.nlp(text)\n",
    "        \n",
    "        features = {\n",
    "            'tokens': [token.text for token in doc],\n",
    "            'lemmas': [token.lemma_ for token in doc if token.is_alpha],\n",
    "            'pos_tags': [token.pos_ for token in doc],\n",
    "            'dep_tags': [token.dep_ for token in doc],\n",
    "            'is_stop': [token.is_stop for token in doc],\n",
    "            'is_alpha': [token.is_alpha for token in doc],\n",
    "            'is_punct': [token.is_punct for token in doc]\n",
    "        }\n",
    "        \n",
    "        return features, doc\n",
    "    \n",
    "    def direct_preprocessing(self, text):\n",
    "        \"\"\" For direct copy detection \"\"\"\n",
    "        return text.lower().strip()\n",
    "    \n",
    "    def semantic_preprocessing(self, text):\n",
    "        \"\"\" For semantic similarity \"\"\"\n",
    "        # Remove noise like stop words\n",
    "        doc = self.nlp(text)\n",
    "        processed_tokens = []\n",
    "        for token in doc:\n",
    "            if not token.is_punct and not token.is_space and token.is_alpha and not token.is_stop:\n",
    "                processed_tokens.append(token.lemma_.lower())\n",
    "        return \" \".join(processed_tokens)\n",
    "    \n",
    "    def syntactic_preprocessing(self, text):\n",
    "        \"\"\" For syntactic similarity \"\"\"\n",
    "        # Normalize\n",
    "        doc = self.nlp(text)\n",
    "        processed_tokens = []\n",
    "        for token in doc:\n",
    "            if token.is_space:\n",
    "                continue\n",
    "            elif token.is_punct:\n",
    "                processed_tokens.append(token.text)\n",
    "            elif token.is_stop:\n",
    "                processed_tokens.append(token.lemma_.lower())\n",
    "            else:\n",
    "                processed_tokens.append(token.lemma_.lower())\n",
    "        return \" \".join(processed_tokens)\n",
    "\n",
    "\n",
    "# Initialize preprocessor\n",
    "preprocessor = TextPreprocessor()\n",
    "\n",
    "# Analyze all sentences\n",
    "print(\"Linguistic Analysis\")\n",
    "print(\"=\" * 60)\n",
    "\n",
    "for i, sentence in enumerate(test_sentences):\n",
    "    print(f\"\\n{i+1:2d}. {'ORIGINAL: ':<20}'{sentence}'\")\n",
    "    features, doc = preprocessor.extract_linguistic_features(sentence)\n",
    "    \n",
    "    print(f\"    {'Direct:     ':<20}'{preprocessor.direct_preprocessing(sentence)}'\")\n",
    "    print(f\"    {'Semantic:   ':<20}'{preprocessor.semantic_preprocessing(sentence)}'\")\n",
    "    print(f\"    {'Syntactic:  ':<20}'{preprocessor.syntactic_preprocessing(sentence)}'\")\n",
    "    \n",
    "    # Basic counts\n",
    "    print(f\"    {'Tokens: ':<20}{len(features['tokens']):2d} | \"\n",
    "          f\"Alpha: {sum(features['is_alpha']):2d} | \"\n",
    "          f\"Stopwords: {sum(features['is_stop']):2d} | \"\n",
    "          f\"Punctuation: {sum(features['is_punct']):2d}\")\n",
    "    \n",
    "    # POS distribution\n",
    "    pos_counts = Counter(features['pos_tags'])\n",
    "    print(f\"    {'POS:':<20}{dict(pos_counts)}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linguistic Analysis\n",
      "============================================================\n",
      "\n",
      " 1. ORIGINAL:           'The cat sat on the mat.'\n",
      "    Direct:             'the cat sat on the mat.'\n",
      "    Semantic:           'cat sit mat'\n",
      "    Syntactic:          'the cat sit on the mat .'\n",
      "    Tokens:              7 | Alpha:  6 | Stopwords:  3 | Punctuation:  1\n",
      "    POS:                {'DET': 2, 'NOUN': 2, 'VERB': 1, 'ADP': 1, 'PUNCT': 1}\n",
      "\n",
      " 2. ORIGINAL:           'The cat sat on the mat'\n",
      "    Direct:             'the cat sat on the mat'\n",
      "    Semantic:           'cat sit mat'\n",
      "    Syntactic:          'the cat sit on the mat'\n",
      "    Tokens:              6 | Alpha:  6 | Stopwords:  3 | Punctuation:  0\n",
      "    POS:                {'DET': 2, 'NOUN': 2, 'VERB': 1, 'ADP': 1}\n",
      "\n",
      " 3. ORIGINAL:           'The  cat  sat  on  the  mat.'\n",
      "    Direct:             'the  cat  sat  on  the  mat.'\n",
      "    Semantic:           'cat sit mat'\n",
      "    Syntactic:          'the cat sit on the mat .'\n",
      "    Tokens:             12 | Alpha:  6 | Stopwords:  3 | Punctuation:  1\n",
      "    POS:                {'DET': 2, 'SPACE': 5, 'NOUN': 2, 'VERB': 1, 'ADP': 1, 'PUNCT': 1}\n",
      "\n",
      " 4. ORIGINAL:           'On the mat, the cat was sitting.'\n",
      "    Direct:             'on the mat, the cat was sitting.'\n",
      "    Semantic:           'mat cat sit'\n",
      "    Syntactic:          'on the mat , the cat be sit .'\n",
      "    Tokens:              9 | Alpha:  7 | Stopwords:  4 | Punctuation:  2\n",
      "    POS:                {'ADP': 1, 'DET': 2, 'NOUN': 2, 'PUNCT': 2, 'AUX': 1, 'VERB': 1}\n",
      "\n",
      " 5. ORIGINAL:           'The feline rested on the rug.'\n",
      "    Direct:             'the feline rested on the rug.'\n",
      "    Semantic:           'feline rest rug'\n",
      "    Syntactic:          'the feline rest on the rug .'\n",
      "    Tokens:              7 | Alpha:  6 | Stopwords:  3 | Punctuation:  1\n",
      "    POS:                {'DET': 2, 'NOUN': 2, 'VERB': 1, 'ADP': 1, 'PUNCT': 1}\n",
      "\n",
      " 6. ORIGINAL:           'The quick brown fox jumps over the lazy dog.'\n",
      "    Direct:             'the quick brown fox jumps over the lazy dog.'\n",
      "    Semantic:           'quick brown fox jump lazy dog'\n",
      "    Syntactic:          'the quick brown fox jump over the lazy dog .'\n",
      "    Tokens:             10 | Alpha:  9 | Stopwords:  3 | Punctuation:  1\n",
      "    POS:                {'DET': 2, 'ADJ': 3, 'PROPN': 1, 'VERB': 1, 'ADP': 1, 'NOUN': 1, 'PUNCT': 1}\n",
      "\n",
      " 7. ORIGINAL:           'Over the lazy dog jumps the quick brown fox.'\n",
      "    Direct:             'over the lazy dog jumps the quick brown fox.'\n",
      "    Semantic:           'lazy dog jump quick brown fox'\n",
      "    Syntactic:          'over the lazy dog jump the quick brown fox .'\n",
      "    Tokens:             10 | Alpha:  9 | Stopwords:  3 | Punctuation:  1\n",
      "    POS:                {'ADP': 1, 'DET': 2, 'ADJ': 3, 'NOUN': 1, 'VERB': 1, 'PROPN': 1, 'PUNCT': 1}\n",
      "\n",
      " 8. ORIGINAL:           'The dog ran in the park.'\n",
      "    Direct:             'the dog ran in the park.'\n",
      "    Semantic:           'dog run park'\n",
      "    Syntactic:          'the dog run in the park .'\n",
      "    Tokens:              7 | Alpha:  6 | Stopwords:  3 | Punctuation:  1\n",
      "    POS:                {'DET': 2, 'NOUN': 2, 'VERB': 1, 'ADP': 1, 'PUNCT': 1}\n",
      "\n",
      " 9. ORIGINAL:           'I love programming.'\n",
      "    Direct:             'i love programming.'\n",
      "    Semantic:           'love programming'\n",
      "    Syntactic:          'i love programming .'\n",
      "    Tokens:              4 | Alpha:  3 | Stopwords:  1 | Punctuation:  1\n",
      "    POS:                {'PRON': 1, 'VERB': 1, 'NOUN': 1, 'PUNCT': 1}\n",
      "\n",
      "10. ORIGINAL:           'She enjoys reading books.'\n",
      "    Direct:             'she enjoys reading books.'\n",
      "    Semantic:           'enjoy read book'\n",
      "    Syntactic:          'she enjoy read book .'\n",
      "    Tokens:              5 | Alpha:  4 | Stopwords:  1 | Punctuation:  1\n",
      "    POS:                {'PRON': 1, 'VERB': 2, 'NOUN': 1, 'PUNCT': 1}\n",
      "\n",
      "11. ORIGINAL:           'Short.'\n",
      "    Direct:             'short.'\n",
      "    Semantic:           'short'\n",
      "    Syntactic:          'short .'\n",
      "    Tokens:              2 | Alpha:  1 | Stopwords:  0 | Punctuation:  1\n",
      "    POS:                {'ADJ': 1, 'PUNCT': 1}\n",
      "\n",
      "12. ORIGINAL:           'A B C D E F G'\n",
      "    Direct:             'a b c d e f g'\n",
      "    Semantic:           'b c d e f g'\n",
      "    Syntactic:          'a b c d e f g'\n",
      "    Tokens:              7 | Alpha:  7 | Stopwords:  1 | Punctuation:  0\n",
      "    POS:                {'DET': 1, 'NOUN': 2, 'PROPN': 4}\n",
      "\n",
      "13. ORIGINAL:           ''\n",
      "    Direct:             ''\n",
      "    Semantic:           ''\n",
      "    Syntactic:          ''\n",
      "    Tokens:              0 | Alpha:  0 | Stopwords:  0 | Punctuation:  0\n",
      "    POS:                {}\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "id": "83fc18c9de2e354",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:45.255452907Z",
     "start_time": "2025-12-14T01:12:44.578417569Z"
    }
   },
   "source": [
    "\n",
    "def analyze_dependency_structure(text, sentence_id):\n",
    "    \"\"\"Analyze and display dependency parse information\"\"\"\n",
    "    if not text.strip():\n",
    "        print(f\"Sentence {sentence_id}: [EMPTY]\")\n",
    "        return None\n",
    "    \n",
    "    doc = nlp(text)\n",
    "    \n",
    "    print(f\"\\nSentence {sentence_id}: '{text}'\")\n",
    "    print(\"Dependency Parse:\")\n",
    "    print(\"-\" * 50)\n",
    "    \n",
    "    # Extract dependency relationships\n",
    "    dependencies = []\n",
    "    for token in doc:\n",
    "        dependencies.append({\n",
    "            'token': token.text,\n",
    "            'lemma': token.lemma_,\n",
    "            'pos': token.pos_,\n",
    "            'dep': token.dep_,\n",
    "            'head': token.head.text,\n",
    "            'children': [child.text for child in token.children]\n",
    "        })\n",
    "        print(f\"{token.text:<12} {token.dep_:<12} {token.head.text:<12} {[child.text for child in token.children]}\")\n",
    "    \n",
    "    return dependencies\n",
    "\n",
    "def visualize_dependency_tree(text, sentence_id):\n",
    "    \"\"\"Create dependency tree visualization\"\"\"\n",
    "    if not text.strip():\n",
    "        return\n",
    "    \n",
    "    doc = nlp(text)\n",
    "    \n",
    "    # Use displacy for visualization\n",
    "    from spacy import displacy\n",
    "    \n",
    "    print(f\"\\nDependency Tree Visualization - Sentence {sentence_id}:\")\n",
    "    html = displacy.render(doc, style=\"dep\", jupyter=True, options={'distance': 120})\n",
    "    \n",
    "    return html\n",
    "\n",
    "# Analyze dependency structures for key sentence pairs\n",
    "key_pairs = [\n",
    "    (0, \"The cat sat on the mat.\"),\n",
    "    (3, \"On the mat, the cat was sitting.\"),\n",
    "    (4, \"The feline rested on the rug.\"),\n",
    "    (5, \"The quick brown fox jumps over the lazy dog.\")\n",
    "]\n",
    "\n",
    "for sentence_id, sentence in key_pairs:\n",
    "    deps = analyze_dependency_structure(sentence, sentence_id)\n",
    "    visualize_dependency_tree(sentence, sentence_id)\n",
    "    print(\"\\n\" + \"=\"*70 + \"\\n\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Sentence 0: 'The cat sat on the mat.'\n",
      "Dependency Parse:\n",
      "--------------------------------------------------\n",
      "The          det          cat          []\n",
      "cat          nsubj        sat          ['The']\n",
      "sat          ROOT         sat          ['cat', 'on', '.']\n",
      "on           prep         sat          ['mat']\n",
      "the          det          mat          []\n",
      "mat          pobj         on           ['the']\n",
      ".            punct        sat          []\n",
      "\n",
      "Dependency Tree Visualization - Sentence 0:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "<span class=\"tex2jax_ignore\"><svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xml:lang=\"en\" id=\"4f1a6c5bcb604942971e087de6b170e2-0\" class=\"displacy\" width=\"770\" height=\"257.0\" direction=\"ltr\" style=\"max-width: none; height: 257.0px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr\">\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"50\">The</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"50\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"170\">cat</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"170\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"290\">sat</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"290\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"410\">on</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"410\">ADP</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"530\">the</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"530\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"650\">mat.</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"650\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4f1a6c5bcb604942971e087de6b170e2-0-0\" stroke-width=\"2px\" d=\"M70,122.0 C70,62.0 165.0,62.0 165.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4f1a6c5bcb604942971e087de6b170e2-0-0\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M70,124.0 L62,112.0 78,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4f1a6c5bcb604942971e087de6b170e2-0-1\" stroke-width=\"2px\" d=\"M190,122.0 C190,62.0 285.0,62.0 285.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4f1a6c5bcb604942971e087de6b170e2-0-1\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nsubj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M190,124.0 L182,112.0 198,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4f1a6c5bcb604942971e087de6b170e2-0-2\" stroke-width=\"2px\" d=\"M310,122.0 C310,62.0 405.0,62.0 405.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4f1a6c5bcb604942971e087de6b170e2-0-2\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">prep</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M405.0,124.0 L413.0,112.0 397.0,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4f1a6c5bcb604942971e087de6b170e2-0-3\" stroke-width=\"2px\" d=\"M550,122.0 C550,62.0 645.0,62.0 645.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4f1a6c5bcb604942971e087de6b170e2-0-3\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M550,124.0 L542,112.0 558,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-4f1a6c5bcb604942971e087de6b170e2-0-4\" stroke-width=\"2px\" d=\"M430,122.0 C430,2.0 650.0,2.0 650.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-4f1a6c5bcb604942971e087de6b170e2-0-4\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">pobj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M650.0,124.0 L658.0,112.0 642.0,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "</svg></span>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======================================================================\n",
      "\n",
      "\n",
      "Sentence 3: 'On the mat, the cat was sitting.'\n",
      "Dependency Parse:\n",
      "--------------------------------------------------\n",
      "On           prep         sitting      ['mat']\n",
      "the          det          mat          []\n",
      "mat          pobj         On           ['the']\n",
      ",            punct        sitting      []\n",
      "the          det          cat          []\n",
      "cat          nsubj        sitting      ['the']\n",
      "was          aux          sitting      []\n",
      "sitting      ROOT         sitting      ['On', ',', 'cat', 'was', '.']\n",
      ".            punct        sitting      []\n",
      "\n",
      "Dependency Tree Visualization - Sentence 3:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "<span class=\"tex2jax_ignore\"><svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xml:lang=\"en\" id=\"9859b04d54454345853c2e92190cd28a-0\" class=\"displacy\" width=\"890\" height=\"317.0\" direction=\"ltr\" style=\"max-width: none; height: 317.0px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr\">\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"50\">On</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"50\">ADP</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"170\">the</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"170\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"290\">mat,</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"290\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"410\">the</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"410\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"530\">cat</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"530\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"650\">was</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"650\">AUX</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"770\">sitting.</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"770\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-9859b04d54454345853c2e92190cd28a-0-0\" stroke-width=\"2px\" d=\"M70,182.0 C70,2.0 770.0,2.0 770.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-9859b04d54454345853c2e92190cd28a-0-0\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">prep</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M70,184.0 L62,172.0 78,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-9859b04d54454345853c2e92190cd28a-0-1\" stroke-width=\"2px\" d=\"M190,182.0 C190,122.0 280.0,122.0 280.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-9859b04d54454345853c2e92190cd28a-0-1\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M190,184.0 L182,172.0 198,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-9859b04d54454345853c2e92190cd28a-0-2\" stroke-width=\"2px\" d=\"M70,182.0 C70,62.0 285.0,62.0 285.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-9859b04d54454345853c2e92190cd28a-0-2\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">pobj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M285.0,184.0 L293.0,172.0 277.0,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-9859b04d54454345853c2e92190cd28a-0-3\" stroke-width=\"2px\" d=\"M430,182.0 C430,122.0 520.0,122.0 520.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-9859b04d54454345853c2e92190cd28a-0-3\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M430,184.0 L422,172.0 438,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-9859b04d54454345853c2e92190cd28a-0-4\" stroke-width=\"2px\" d=\"M550,182.0 C550,62.0 765.0,62.0 765.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-9859b04d54454345853c2e92190cd28a-0-4\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nsubj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M550,184.0 L542,172.0 558,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-9859b04d54454345853c2e92190cd28a-0-5\" stroke-width=\"2px\" d=\"M670,182.0 C670,122.0 760.0,122.0 760.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-9859b04d54454345853c2e92190cd28a-0-5\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">aux</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M670,184.0 L662,172.0 678,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "</svg></span>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======================================================================\n",
      "\n",
      "\n",
      "Sentence 4: 'The feline rested on the rug.'\n",
      "Dependency Parse:\n",
      "--------------------------------------------------\n",
      "The          det          feline       []\n",
      "feline       nsubj        rested       ['The']\n",
      "rested       ROOT         rested       ['feline', 'on', '.']\n",
      "on           prep         rested       ['rug']\n",
      "the          det          rug          []\n",
      "rug          pobj         on           ['the']\n",
      ".            punct        rested       []\n",
      "\n",
      "Dependency Tree Visualization - Sentence 4:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "<span class=\"tex2jax_ignore\"><svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xml:lang=\"en\" id=\"51aaaa18679f4b1e9dd3354d9dfbce26-0\" class=\"displacy\" width=\"770\" height=\"257.0\" direction=\"ltr\" style=\"max-width: none; height: 257.0px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr\">\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"50\">The</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"50\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"170\">feline</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"170\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"290\">rested</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"290\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"410\">on</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"410\">ADP</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"530\">the</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"530\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"167.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"650\">rug.</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"650\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-0\" stroke-width=\"2px\" d=\"M70,122.0 C70,62.0 165.0,62.0 165.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-0\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M70,124.0 L62,112.0 78,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-1\" stroke-width=\"2px\" d=\"M190,122.0 C190,62.0 285.0,62.0 285.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-1\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nsubj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M190,124.0 L182,112.0 198,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-2\" stroke-width=\"2px\" d=\"M310,122.0 C310,62.0 405.0,62.0 405.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-2\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">prep</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M405.0,124.0 L413.0,112.0 397.0,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-3\" stroke-width=\"2px\" d=\"M550,122.0 C550,62.0 645.0,62.0 645.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-3\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M550,124.0 L542,112.0 558,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-4\" stroke-width=\"2px\" d=\"M430,122.0 C430,2.0 650.0,2.0 650.0,122.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-51aaaa18679f4b1e9dd3354d9dfbce26-0-4\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">pobj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M650.0,124.0 L658.0,112.0 642.0,112.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "</svg></span>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======================================================================\n",
      "\n",
      "\n",
      "Sentence 5: 'The quick brown fox jumps over the lazy dog.'\n",
      "Dependency Parse:\n",
      "--------------------------------------------------\n",
      "The          det          fox          []\n",
      "quick        amod         fox          []\n",
      "brown        amod         fox          []\n",
      "fox          nsubj        jumps        ['The', 'quick', 'brown']\n",
      "jumps        ROOT         jumps        ['fox', 'over', '.']\n",
      "over         prep         jumps        ['dog']\n",
      "the          det          dog          []\n",
      "lazy         amod         dog          []\n",
      "dog          pobj         over         ['the', 'lazy']\n",
      ".            punct        jumps        []\n",
      "\n",
      "Dependency Tree Visualization - Sentence 5:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "<span class=\"tex2jax_ignore\"><svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" xml:lang=\"en\" id=\"7957c47681d843aa90b7826d9cd76aea-0\" class=\"displacy\" width=\"1130\" height=\"317.0\" direction=\"ltr\" style=\"max-width: none; height: 317.0px; color: #000000; background: #ffffff; font-family: Arial; direction: ltr\">\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"50\">The</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"50\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"170\">quick</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"170\">ADJ</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"290\">brown</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"290\">ADJ</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"410\">fox</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"410\">PROPN</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"530\">jumps</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"530\">VERB</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"650\">over</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"650\">ADP</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"770\">the</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"770\">DET</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"890\">lazy</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"890\">ADJ</tspan>\n",
       "</text>\n",
       "\n",
       "<text class=\"displacy-token\" fill=\"currentColor\" text-anchor=\"middle\" y=\"227.0\">\n",
       "    <tspan class=\"displacy-word\" fill=\"currentColor\" x=\"1010\">dog.</tspan>\n",
       "    <tspan class=\"displacy-tag\" dy=\"2em\" fill=\"currentColor\" x=\"1010\">NOUN</tspan>\n",
       "</text>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-0\" stroke-width=\"2px\" d=\"M70,182.0 C70,2.0 410.0,2.0 410.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-0\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M70,184.0 L62,172.0 78,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-1\" stroke-width=\"2px\" d=\"M190,182.0 C190,62.0 405.0,62.0 405.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-1\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">amod</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M190,184.0 L182,172.0 198,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-2\" stroke-width=\"2px\" d=\"M310,182.0 C310,122.0 400.0,122.0 400.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-2\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">amod</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M310,184.0 L302,172.0 318,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-3\" stroke-width=\"2px\" d=\"M430,182.0 C430,122.0 520.0,122.0 520.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-3\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">nsubj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M430,184.0 L422,172.0 438,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-4\" stroke-width=\"2px\" d=\"M550,182.0 C550,122.0 640.0,122.0 640.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-4\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">prep</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M640.0,184.0 L648.0,172.0 632.0,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-5\" stroke-width=\"2px\" d=\"M790,182.0 C790,62.0 1005.0,62.0 1005.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-5\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">det</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M790,184.0 L782,172.0 798,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-6\" stroke-width=\"2px\" d=\"M910,182.0 C910,122.0 1000.0,122.0 1000.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-6\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">amod</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M910,184.0 L902,172.0 918,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "\n",
       "<g class=\"displacy-arrow\">\n",
       "    <path class=\"displacy-arc\" id=\"arrow-7957c47681d843aa90b7826d9cd76aea-0-7\" stroke-width=\"2px\" d=\"M670,182.0 C670,2.0 1010.0,2.0 1010.0,182.0\" fill=\"none\" stroke=\"currentColor\"/>\n",
       "    <text dy=\"1.25em\" style=\"font-size: 0.8em; letter-spacing: 1px\">\n",
       "        <textPath xlink:href=\"#arrow-7957c47681d843aa90b7826d9cd76aea-0-7\" class=\"displacy-label\" startOffset=\"50%\" side=\"left\" fill=\"currentColor\" text-anchor=\"middle\">pobj</textPath>\n",
       "    </text>\n",
       "    <path class=\"displacy-arrowhead\" d=\"M1010.0,184.0 L1018.0,172.0 1002.0,172.0\" fill=\"currentColor\"/>\n",
       "</g>\n",
       "</svg></span>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "======================================================================\n",
      "\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "id": "e413238c1af12f62",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:45.762461029Z",
     "start_time": "2025-12-14T01:12:45.271388321Z"
    }
   },
   "source": [
    "def analyze_vocabulary_distribution(sentences):\n",
    "    \"\"\"Analyze word frequencies and vocabulary across sentences\"\"\"\n",
    "    all_words = []\n",
    "    all_lemmas = []\n",
    "    content_words = []\n",
    "    \n",
    "    for sent in sentences:\n",
    "        doc = nlp(sent)\n",
    "        for token in doc:\n",
    "            if token.is_alpha and not token.is_space:\n",
    "                all_words.append(token.text.lower())\n",
    "                all_lemmas.append(token.lemma_.lower())\n",
    "                if not token.is_stop and token.pos_ in ['NOUN', 'VERB', 'ADJ', 'ADV']:\n",
    "                    content_words.append(token.lemma_.lower())\n",
    "    \n",
    "    # Calculate frequencies\n",
    "    word_freq = Counter(all_words)\n",
    "    lemma_freq = Counter(all_lemmas)\n",
    "    content_freq = Counter(content_words)\n",
    "    \n",
    "    return word_freq, lemma_freq, content_freq\n",
    "\n",
    "word_freq, lemma_freq, content_freq = analyze_vocabulary_distribution(test_sentences)\n",
    "\n",
    "print(\"VOCABULARY ANALYSIS\")\n",
    "print(\"=\" * 50)\n",
    "print(f\"Total unique words: {len(word_freq)}\")\n",
    "print(f\"Total unique lemmas: {len(lemma_freq)}\")\n",
    "print(f\"Total content words: {len(content_freq)}\")\n",
    "\n",
    "# Plot most common words\n",
    "fig, axes = plt.subplots(1, 3, figsize=(15, 5))\n",
    "\n",
    "# Original words\n",
    "common_words = word_freq.most_common(10)\n",
    "axes[0].bar([w[0] for w in common_words], [w[1] for w in common_words])\n",
    "axes[0].set_title('Top 10 Most Common Words')\n",
    "axes[0].tick_params(axis='x', rotation=45)\n",
    "\n",
    "# Lemmas\n",
    "common_lemmas = lemma_freq.most_common(10)\n",
    "axes[1].bar([w[0] for w in common_lemmas], [w[1] for w in common_lemmas])\n",
    "axes[1].set_title('Top 10 Most Common Lemmas')\n",
    "axes[1].tick_params(axis='x', rotation=45)\n",
    "\n",
    "# Content words\n",
    "common_content = content_freq.most_common(10)\n",
    "axes[2].bar([w[0] for w in common_content], [w[1] for w in common_content])\n",
    "axes[2].set_title('Top 10 Content Words')\n",
    "axes[2].tick_params(axis='x', rotation=45)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "VOCABULARY ANALYSIS\n",
      "==================================================\n",
      "Total unique words: 35\n",
      "Total unique lemmas: 34\n",
      "Total content words: 21\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 1500x500 with 3 Axes>"
      ],
      "image/png": "[encoded data, 47956 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "id": "6aff51eb71eb2238",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:51.876467760Z",
     "start_time": "2025-12-14T01:12:45.765076317Z"
    }
   },
   "source": [
    "def create_similarity_matrix(sentences, method='direct'):\n",
    "    \"\"\"Create similarity matrix between all sentences\"\"\"\n",
    "    n = len(sentences)\n",
    "    similarity_matrix = np.zeros((n, n))\n",
    "    \n",
    "    preprocessor = TextPreprocessor()\n",
    "    \n",
    "    for i in range(n):\n",
    "        for j in range(n):\n",
    "            if i == j:\n",
    "                similarity_matrix[i][j] = 1.0\n",
    "            else:\n",
    "                if method == 'direct':\n",
    "                    sent1 = preprocessor.direct_preprocessing(sentences[i])\n",
    "                    sent2 = preprocessor.direct_preprocessing(sentences[j])\n",
    "                    # Simple character-based similarity\n",
    "                    if not sent1 or not sent2:\n",
    "                        similarity = 0.0\n",
    "                    else:\n",
    "                        min_len = min(len(sent1), len(sent2))\n",
    "                        max_len = max(len(sent1), len(sent2))\n",
    "                        similarity = min_len / max_len if max_len > 0 else 0.0\n",
    "                \n",
    "                elif method == 'semantic':\n",
    "                    sent1 = preprocessor.semantic_preprocessing(sentences[i])\n",
    "                    sent2 = preprocessor.semantic_preprocessing(sentences[j])\n",
    "                    # Jaccard similarity on semantic tokens\n",
    "                    words1 = set(sent1.split())\n",
    "                    words2 = set(sent2.split())\n",
    "                    if not words1 and not words2:\n",
    "                        similarity = 1.0\n",
    "                    elif not words1 or not words2:\n",
    "                        similarity = 0.0\n",
    "                    else:\n",
    "                        intersection = words1.intersection(words2)\n",
    "                        union = words1.union(words2)\n",
    "                        similarity = len(intersection) / len(union)\n",
    "                \n",
    "                similarity_matrix[i][j] = similarity\n",
    "    \n",
    "    return similarity_matrix\n",
    "\n",
    "# Create and visualize similarity matrices\n",
    "direct_matrix = create_similarity_matrix(test_sentences, 'direct')\n",
    "semantic_matrix = create_similarity_matrix(test_sentences, 'semantic')\n",
    "\n",
    "fig, axes = plt.subplots(1, 2, figsize=(15, 6))\n",
    "\n",
    "# Direct similarity heatmap\n",
    "im1 = axes[0].imshow(direct_matrix, cmap='viridis', aspect='auto', vmin=0, vmax=1)\n",
    "axes[0].set_title('Direct Preprocessing Similarity Matrix')\n",
    "axes[0].set_xlabel('Sentence ID')\n",
    "axes[0].set_ylabel('Sentence ID')\n",
    "plt.colorbar(im1, ax=axes[0])\n",
    "\n",
    "# Semantic similarity heatmap\n",
    "im2 = axes[1].imshow(semantic_matrix, cmap='viridis', aspect='auto', vmin=0, vmax=1)\n",
    "axes[1].set_title('Semantic Preprocessing Similarity Matrix')\n",
    "axes[1].set_xlabel('Sentence ID')\n",
    "axes[1].set_ylabel('Sentence ID')\n",
    "plt.colorbar(im2, ax=axes[1])\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1500x600 with 4 Axes>"
      ],
      "image/png": "[encoded data, 43948 characters]"
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "markdown",
   "id": "f0fc7b35",
   "metadata": {},
   "source": [
    "Key observations:\n",
    "- Direct similarity catches exact copies but misses paraphrases\n",
    "- Semantic similarity identifies meaning-based relationships\n",
    "- Notice how sentences 0-2 (direct copies) are highly similar in both matrices\n",
    "- Sentences 3-4 (paraphrases) show higher semantic similarity"
   ]
  },
  {
   "cell_type": "code",
   "id": "98f16f7d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-12-14T01:12:51.977965137Z",
     "start_time": "2025-12-14T01:12:51.931400526Z"
    }
   },
   "source": [],
   "outputs": [],
   "execution_count": 6
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
