{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T15:58:03.741302Z",
     "start_time": "2025-07-08T15:58:03.737218Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import sys\n",
    "import os\n",
    "import importlib\n",
    "\n",
    "sys.path.append(os.path.abspath(\"..\"))\n",
    "import misc\n",
    "importlib.reload(misc)\n",
    "from misc import load_csv_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "74a8a262bc55d976",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T15:58:27.047640Z",
     "start_time": "2025-07-08T15:58:06.842257Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ">> Loading CSV dataset from names_featured.csv\n",
      ">> Detected fieldnames: ['name', 'sex', 'region', 'year', 'words', 'length', 'probable_native', 'probable_surname', 'identified_category', 'identified_name', 'identified_surname', 'annotated', 'province']\n",
      ">> Successfully loaded with UTF-8 encoding\n"
     ]
    }
   ],
   "source": [
    "df = pd.DataFrame(load_csv_dataset(\"names_featured.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ebb18cabd40b011f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T15:58:50.742471Z",
     "start_time": "2025-07-08T15:58:45.080832Z"
    }
   },
   "outputs": [],
   "source": [
    "df['year'] = pd.to_numeric(df['year'], errors='coerce')\n",
    "df['words'] = pd.to_numeric(df['words'], errors='coerce')\n",
    "df['length'] = pd.to_numeric(df['length'], errors='coerce')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "295eda175081129c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T15:59:02.783458Z",
     "start_time": "2025-07-08T15:59:02.297971Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>year</th>\n",
       "      <td>5522918.0</td>\n",
       "      <td>2017.037288</td>\n",
       "      <td>4.657671</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>2018.0</td>\n",
       "      <td>2021.0</td>\n",
       "      <td>2023.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>words</th>\n",
       "      <td>5522918.0</td>\n",
       "      <td>2.862474</td>\n",
       "      <td>0.473505</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>length</th>\n",
       "      <td>5522918.0</td>\n",
       "      <td>18.217497</td>\n",
       "      <td>3.498697</td>\n",
       "      <td>1.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            count         mean       std     min     25%     50%     75%  \\\n",
       "year    5522918.0  2017.037288  4.657671  2008.0  2013.0  2018.0  2021.0   \n",
       "words   5522918.0     2.862474  0.473505     1.0     3.0     3.0     3.0   \n",
       "length  5522918.0    18.217497  3.498697     1.0    16.0    18.0    21.0   \n",
       "\n",
       "           max  \n",
       "year    2023.0  \n",
       "words     10.0  \n",
       "length    49.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c6f41484d947a77a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T15:59:06.087860Z",
     "start_time": "2025-07-08T15:59:05.834072Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 46652 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from scipy.interpolate import make_interp_spline\n",
    "\n",
    "# Step 1: Count frequencies\n",
    "word_counts = df[\"words\"].value_counts().sort_index()\n",
    "x = word_counts.index.values       # distinct word counts\n",
    "y = word_counts.values             # their frequencies\n",
    "\n",
    "# Step 2: Create smoother X values (interpolation)\n",
    "x_smooth = np.linspace(x.min(), x.max(), 300)\n",
    "spline = make_interp_spline(x, y, k=3)  # cubic spline\n",
    "y_smooth = spline(x_smooth)\n",
    "\n",
    "# Step 3: Plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(x_smooth, y_smooth, color='blue', label='Smoothed Curve')\n",
    "plt.scatter(x, y, color='black', zorder=5, label='Actual Counts')\n",
    "plt.title(\"Smoothed Word Count Distribution\")\n",
    "plt.xlabel(\"Number of Words in Name\")\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.grid(True)\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2533054dd18b57c7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T16:02:00.928164Z",
     "start_time": "2025-07-08T16:02:00.666031Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56640 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from scipy.interpolate import make_interp_spline\n",
    "\n",
    "# Step 1: Count frequencies\n",
    "word_counts = df[\"length\"].value_counts().sort_index()\n",
    "x = word_counts.index.values       # distinct word counts\n",
    "y = word_counts.values             # their frequencies\n",
    "\n",
    "# Step 2: Create smoother X values (interpolation)\n",
    "x_smooth = np.linspace(x.min(), x.max(), 300)\n",
    "spline = make_interp_spline(x, y, k=3)  # cubic spline\n",
    "y_smooth = spline(x_smooth)\n",
    "\n",
    "# Step 3: Plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "plt.plot(x_smooth, y_smooth, color='blue', label='Smoothed Curve')\n",
    "plt.scatter(x, y, color='black', zorder=5, label='Actual Counts')\n",
    "plt.title(\"Smoothed Word Count Distribution\")\n",
    "plt.xlabel(\"Number of Words in Name\")\n",
    "plt.ylabel(\"Frequency\")\n",
    "plt.grid(True)\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "af313d893f31698f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T16:08:48.468235Z",
     "start_time": "2025-07-08T16:08:47.627659Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ">> Using categorical units to plot a list of strings that are all parsable as floats or dates. If these strings should be plotted as numbers, cast to the appropriate data type before plotting.\n",
      ">> Using categorical units to plot a list of strings that are all parsable as floats or dates. If these strings should be plotted as numbers, cast to the appropriate data type before plotting.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 31988 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gender_by_year = df.groupby(['year', 'sex']).size().reset_index(name='count')\n",
    "\n",
    "plt.figure(figsize=(12, 6))\n",
    "sns.barplot(data=gender_by_year, x='year', y='count', hue='sex')\n",
    "plt.title('Gender Count by Year')\n",
    "plt.xlabel('Year')\n",
    "plt.ylabel('Count')\n",
    "plt.xticks(rotation=45)\n",
    "plt.legend(title='Gender')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "93232202305056a3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T16:09:45.349954Z",
     "start_time": "2025-07-08T16:09:45.086546Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 26676 characters]",
      "text/plain": [
       "<Figure size 800x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sex_counts = df['sex'].value_counts()\n",
    "\n",
    "plt.figure(figsize=(8, 8))\n",
    "plt.pie(sex_counts, labels=sex_counts.index, autopct='%1.1f%%', startangle=90, colors=['lightblue', 'lightpink'])\n",
    "plt.title('Distribution of Sex')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f7b8afb3bc565875",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T16:13:37.061494Z",
     "start_time": "2025-07-08T16:13:36.764013Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24356 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(df['words'], bins=30, color='skyblue', edgecolor='black')\n",
    "plt.title('Histogram of Word Counts')\n",
    "plt.xlabel('Word Counts')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f5c322d32b2d60fa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T16:13:42.022402Z",
     "start_time": "2025-07-08T16:13:41.781236Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 24696 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "plt.hist(df['length'], bins=30, color='lightgreen', edgecolor='black')\n",
    "plt.title('Histogram of Name Lengths')\n",
    "plt.xlabel('Name Lengths')\n",
    "plt.ylabel('Frequency')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "cc849ca9d9d43ae3",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T16:14:08.587040Z",
     "start_time": "2025-07-08T16:13:47.209322Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 27600 characters]",
      "text/plain": [
       "<Figure size 800x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 6))\n",
    "sns.boxplot(data=df[['words', 'length']])\n",
    "plt.title('Boxplots of Word Counts and Name Lengths')\n",
    "plt.ylabel('Values')\n",
    "plt.xticks(ticks=[0, 1], labels=['Words', 'Length'])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f27c59dd8b8eaf4b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-08T18:18:51.166573Z",
     "start_time": "2025-07-08T18:18:42.805191Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 34488 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(data=df, x='length', y='words', alpha=0.7)\n",
    "plt.title('Scatterplot: Name Length vs Word Counts')\n",
    "plt.xlabel('Name Length')\n",
    "plt.ylabel('Word Counts')\n",
    "plt.grid(True)\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
