{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "291bb40a",
   "metadata": {},
   "source": [
    "# 🤖 ML Pipeline: Dự Đoán Số Lượng Nhân Sự\n",
    "\n",
    "**Mục tiêu:** Dự đoán số lượng nhân sự (`so_luong`) cần thiết cho mỗi ca làm việc\n",
    "\n",
    "**Dataset:** FINAL_DATASET_WITH_TEXT.xlsx (454 rows × 51 columns)\n",
    "\n",
    "**Features sử dụng:** 47 features (không bao gồm 2 cột text - để lại cho phase sau)\n",
    "\n",
    "**Target:** `so_luong` (0-64 nhân sự)\n",
    "\n",
    "---\n",
    "\n",
    "## 📋 Nội dung Notebook:\n",
    "\n",
    "1. ✅ Import Libraries & Load Data\n",
    "2. ✅ Exploratory Data Analysis (EDA)\n",
    "3. ✅ Data Cleaning & Preprocessing  \n",
    "4. ✅ Feature Engineering\n",
    "5. ✅ Visualizations & Insights\n",
    "6. ✅ Train/Val/Test Split\n",
    "7. ✅ Baseline Models\n",
    "8. ✅ Advanced Models (RF, XGBoost, LightGBM)\n",
    "9. ✅ Model Evaluation & Comparison\n",
    "10. ✅ Feature Importance Analysis\n",
    "11. ✅ Final Results & Recommendations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d2dd2408",
   "metadata": {},
   "source": [
    "# 1️⃣ Import Libraries & Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9ada57ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ Libraries imported successfully!\n",
      "📅 Date: 2026-01-05 23:29:39\n"
     ]
    }
   ],
   "source": [
    "# Import essential libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "# Machine Learning libraries\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
    "from sklearn.preprocessing import StandardScaler, LabelEncoder, OneHotEncoder\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor\n",
    "from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
    "import xgboost as xgb\n",
    "import lightgbm as lgb\n",
    "\n",
    "# Configuration\n",
    "warnings.filterwarnings('ignore')\n",
    "plt.style.use('seaborn-v0_8-darkgrid')\n",
    "sns.set_palette(\"husl\")\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_rows', 100)\n",
    "\n",
    "# Random seed for reproducibility\n",
    "RANDOM_STATE = 42\n",
    "np.random.seed(RANDOM_STATE)\n",
    "\n",
    "# Create folders for outputs\n",
    "os.makedirs('data/cleaned', exist_ok=True)\n",
    "os.makedirs('data/splits', exist_ok=True)\n",
    "os.makedirs('models', exist_ok=True)\n",
    "os.makedirs('plots', exist_ok=True)\n",
    "\n",
    "print(\"✅ Libraries imported successfully!\")\n",
    "print(f\"📅 Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "0350590f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "📂 Loading dataset...\n",
      "✅ Dataset loaded successfully!\n",
      "📊 Shape: (454, 51)\n",
      "📋 Columns: ['ma_dia_diem', 'all_task_normal', 'all_task_dinhky', 'loai_ca', 'bat_dau', 'ket_thuc', 'tong_gio_lam', 'so_ca_cua_toa', 'so_luong', 'num_tasks']... (showing first 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ma_dia_diem</th>\n",
       "      <th>all_task_normal</th>\n",
       "      <th>all_task_dinhky</th>\n",
       "      <th>loai_ca</th>\n",
       "      <th>bat_dau</th>\n",
       "      <th>ket_thuc</th>\n",
       "      <th>tong_gio_lam</th>\n",
       "      <th>so_ca_cua_toa</th>\n",
       "      <th>so_luong</th>\n",
       "      <th>num_tasks</th>\n",
       "      <th>num_cleaning_tasks</th>\n",
       "      <th>num_trash_collection_tasks</th>\n",
       "      <th>num_monitoring_tasks</th>\n",
       "      <th>num_room_cleaning_tasks</th>\n",
       "      <th>num_deep_cleaning_tasks</th>\n",
       "      <th>num_maintenance_tasks</th>\n",
       "      <th>num_support_tasks</th>\n",
       "      <th>num_other_tasks</th>\n",
       "      <th>num_wc_tasks</th>\n",
       "      <th>num_hallway_tasks</th>\n",
       "      <th>num_lobby_tasks</th>\n",
       "      <th>num_patient_room_tasks</th>\n",
       "      <th>num_clinic_room_tasks</th>\n",
       "      <th>num_surgery_room_tasks</th>\n",
       "      <th>num_outdoor_tasks</th>\n",
       "      <th>num_elevator_tasks</th>\n",
       "      <th>num_office_tasks</th>\n",
       "      <th>num_technical_room_tasks</th>\n",
       "      <th>cleaning_ratio</th>\n",
       "      <th>trash_collection_ratio</th>\n",
       "      <th>monitoring_ratio</th>\n",
       "      <th>room_cleaning_ratio</th>\n",
       "      <th>area_diversity</th>\n",
       "      <th>task_complexity_score</th>\n",
       "      <th>loai_hinh</th>\n",
       "      <th>ten_toa_thap</th>\n",
       "      <th>muc_do_luu_luong</th>\n",
       "      <th>so_tang</th>\n",
       "      <th>so_cua_thang_may</th>\n",
       "      <th>dien_tich_ngoai_canh</th>\n",
       "      <th>dien_tich_sanh</th>\n",
       "      <th>dien_tich_hanh_lang</th>\n",
       "      <th>dien_tich_wc</th>\n",
       "      <th>dien_tich_phong</th>\n",
       "      <th>dien_tich_tham</th>\n",
       "      <th>doc_ham</th>\n",
       "      <th>vien_phan_quang</th>\n",
       "      <th>op_tuong</th>\n",
       "      <th>op_chan_tuong</th>\n",
       "      <th>ranh_thoat_nuoc</th>\n",
       "      <th>dien_tich_kinh</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115-2</td>\n",
       "      <td>Làm sạch toàn bộ phòng giao dịch tầng 1 (kể cả...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Part time</td>\n",
       "      <td>06:30:00</td>\n",
       "      <td>10:30:00</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.1429</td>\n",
       "      <td>0.2857</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.38</td>\n",
       "      <td>0</td>\n",
       "      <td>AGRIBANK CHI NHÁNH MỸ ĐÌNH</td>\n",
       "      <td>Trung bình (11–20 người)</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>20.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>115-5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Hành chính</td>\n",
       "      <td>06:30:00</td>\n",
       "      <td>16:00:00</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>AGRIBANK PGD SỐ 05</td>\n",
       "      <td>Trung bình (11–20 người)</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>300.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>101-1</td>\n",
       "      <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
       "      <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
       "      <td>Hành chính</td>\n",
       "      <td>06:30:00</td>\n",
       "      <td>16:00:00</td>\n",
       "      <td>7.5</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>441.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.585</td>\n",
       "      <td>0.3288</td>\n",
       "      <td>0.3039</td>\n",
       "      <td>0.1474</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0</td>\n",
       "      <td>Tòa 5 tầng</td>\n",
       "      <td>Rất cao (Trên 40 người)</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>348.0</td>\n",
       "      <td>6825.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>9176.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>25</td>\n",
       "      <td>894.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101-1</td>\n",
       "      <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
       "      <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
       "      <td>Ca sáng</td>\n",
       "      <td>06:00:00</td>\n",
       "      <td>14:00:00</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>441.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.585</td>\n",
       "      <td>0.3288</td>\n",
       "      <td>0.3039</td>\n",
       "      <td>0.1474</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0</td>\n",
       "      <td>Tòa 5 tầng</td>\n",
       "      <td>Rất cao (Trên 40 người)</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>348.0</td>\n",
       "      <td>6825.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>9176.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>25</td>\n",
       "      <td>894.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>101-1</td>\n",
       "      <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
       "      <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
       "      <td>Ca chiều</td>\n",
       "      <td>14:00:00</td>\n",
       "      <td>22:00:00</td>\n",
       "      <td>8.0</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>441.0</td>\n",
       "      <td>258.0</td>\n",
       "      <td>145.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>90.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>41.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>0.585</td>\n",
       "      <td>0.3288</td>\n",
       "      <td>0.3039</td>\n",
       "      <td>0.1474</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0</td>\n",
       "      <td>Tòa 5 tầng</td>\n",
       "      <td>Rất cao (Trên 40 người)</td>\n",
       "      <td>10</td>\n",
       "      <td>18</td>\n",
       "      <td>1700.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2600.0</td>\n",
       "      <td>348.0</td>\n",
       "      <td>6825.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>70</td>\n",
       "      <td>0</td>\n",
       "      <td>9176.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>25</td>\n",
       "      <td>894.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ma_dia_diem                                    all_task_normal  \\\n",
       "0       115-2  Làm sạch toàn bộ phòng giao dịch tầng 1 (kể cả...   \n",
       "1       115-5                                                NaN   \n",
       "2       101-1  Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...   \n",
       "3       101-1  Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...   \n",
       "4       101-1  Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...   \n",
       "\n",
       "                                     all_task_dinhky     loai_ca   bat_dau  \\\n",
       "0                                                NaN   Part time  06:30:00   \n",
       "1                                                NaN  Hành chính  06:30:00   \n",
       "2  Lau bảng biển, bình cứu hỏa , cây nước hành la...  Hành chính  06:30:00   \n",
       "3  Lau bảng biển, bình cứu hỏa , cây nước hành la...     Ca sáng  06:00:00   \n",
       "4  Lau bảng biển, bình cứu hỏa , cây nước hành la...    Ca chiều  14:00:00   \n",
       "\n",
       "   ket_thuc  tong_gio_lam  so_ca_cua_toa  so_luong  num_tasks  \\\n",
       "0  10:30:00           4.0              1         1        7.0   \n",
       "1  16:00:00           8.0              1         1        NaN   \n",
       "2  16:00:00           7.5              6        24      441.0   \n",
       "3  14:00:00           8.0              6         3      441.0   \n",
       "4  22:00:00           8.0              6         5      441.0   \n",
       "\n",
       "   num_cleaning_tasks  num_trash_collection_tasks  num_monitoring_tasks  \\\n",
       "0                 7.0                         1.0                   2.0   \n",
       "1                 NaN                         NaN                   NaN   \n",
       "2               258.0                       145.0                 134.0   \n",
       "3               258.0                       145.0                 134.0   \n",
       "4               258.0                       145.0                 134.0   \n",
       "\n",
       "   num_room_cleaning_tasks  num_deep_cleaning_tasks  num_maintenance_tasks  \\\n",
       "0                      0.0                      1.0                    1.0   \n",
       "1                      NaN                      NaN                    NaN   \n",
       "2                     65.0                     75.0                   62.0   \n",
       "3                     65.0                     75.0                   62.0   \n",
       "4                     65.0                     75.0                   62.0   \n",
       "\n",
       "   num_support_tasks  num_other_tasks  num_wc_tasks  num_hallway_tasks  \\\n",
       "0                0.0              0.0           4.0                0.0   \n",
       "1                NaN              NaN           NaN                NaN   \n",
       "2               57.0             45.0          89.0               90.0   \n",
       "3               57.0             45.0          89.0               90.0   \n",
       "4               57.0             45.0          89.0               90.0   \n",
       "\n",
       "   num_lobby_tasks  num_patient_room_tasks  num_clinic_room_tasks  \\\n",
       "0              0.0                     0.0                    0.0   \n",
       "1              NaN                     NaN                    NaN   \n",
       "2              5.0                    41.0                   25.0   \n",
       "3              5.0                    41.0                   25.0   \n",
       "4              5.0                    41.0                   25.0   \n",
       "\n",
       "   num_surgery_room_tasks  num_outdoor_tasks  num_elevator_tasks  \\\n",
       "0                     0.0                2.0                 3.0   \n",
       "1                     NaN                NaN                 NaN   \n",
       "2                    30.0                4.0                12.0   \n",
       "3                    30.0                4.0                12.0   \n",
       "4                    30.0                4.0                12.0   \n",
       "\n",
       "   num_office_tasks  num_technical_room_tasks  cleaning_ratio  \\\n",
       "0               1.0                       0.0           1.000   \n",
       "1               NaN                       NaN             NaN   \n",
       "2              39.0                      16.0           0.585   \n",
       "3              39.0                      16.0           0.585   \n",
       "4              39.0                      16.0           0.585   \n",
       "\n",
       "   trash_collection_ratio  monitoring_ratio  room_cleaning_ratio  \\\n",
       "0                  0.1429            0.2857               0.0000   \n",
       "1                     NaN               NaN                  NaN   \n",
       "2                  0.3288            0.3039               0.1474   \n",
       "3                  0.3288            0.3039               0.1474   \n",
       "4                  0.3288            0.3039               0.1474   \n",
       "\n",
       "   area_diversity  task_complexity_score  loai_hinh  \\\n",
       "0             4.0                   1.38          0   \n",
       "1             NaN                    NaN          0   \n",
       "2            10.0                  10.00          0   \n",
       "3            10.0                  10.00          0   \n",
       "4            10.0                  10.00          0   \n",
       "\n",
       "                 ten_toa_thap          muc_do_luu_luong  so_tang  \\\n",
       "0  AGRIBANK CHI NHÁNH MỸ ĐÌNH  Trung bình (11–20 người)        4   \n",
       "1          AGRIBANK PGD SỐ 05  Trung bình (11–20 người)        1   \n",
       "2                  Tòa 5 tầng   Rất cao (Trên 40 người)       10   \n",
       "3                  Tòa 5 tầng   Rất cao (Trên 40 người)       10   \n",
       "4                  Tòa 5 tầng   Rất cao (Trên 40 người)       10   \n",
       "\n",
       "   so_cua_thang_may  dien_tich_ngoai_canh  dien_tich_sanh  \\\n",
       "0                 0                   0.0             0.0   \n",
       "1                 0                   0.0             0.0   \n",
       "2                18                1700.0             0.0   \n",
       "3                18                1700.0             0.0   \n",
       "4                18                1700.0             0.0   \n",
       "\n",
       "   dien_tich_hanh_lang  dien_tich_wc  dien_tich_phong  dien_tich_tham  \\\n",
       "0                  0.0          15.0            290.0             0.0   \n",
       "1                  0.0          15.0            300.0             0.0   \n",
       "2               2600.0         348.0           6825.0             0.0   \n",
       "3               2600.0         348.0           6825.0             0.0   \n",
       "4               2600.0         348.0           6825.0             0.0   \n",
       "\n",
       "   doc_ham  vien_phan_quang  op_tuong  op_chan_tuong  ranh_thoat_nuoc  \\\n",
       "0        0                0       0.0            0.0                0   \n",
       "1        0                0       0.0            0.0                0   \n",
       "2       70                0    9176.0           89.0               25   \n",
       "3       70                0    9176.0           89.0               25   \n",
       "4       70                0    9176.0           89.0               25   \n",
       "\n",
       "   dien_tich_kinh  \n",
       "0            20.0  \n",
       "1            30.0  \n",
       "2           894.0  \n",
       "3           894.0  \n",
       "4           894.0  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dataset\n",
    "print(\"📂 Loading dataset...\")\n",
    "df = pd.read_excel('FINAL_DATASET_WITH_TEXT_BACKUP_20260105_213507.xlsx')\n",
    "\n",
    "print(f\"✅ Dataset loaded successfully!\")\n",
    "print(f\"📊 Shape: {df.shape}\")\n",
    "print(f\"📋 Columns: {df.columns.tolist()[:10]}... (showing first 10)\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e99e108",
   "metadata": {},
   "source": [
    "# 2️⃣ Exploratory Data Analysis (EDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8e92132d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "📊 EXPLORATORY DATA ANALYSIS\n",
      "================================================================================\n",
      "\n",
      "1. DATA INFO:\n",
      "   Shape: (454, 51)\n",
      "   Memory: 9.10 MB\n",
      "\n",
      "2. DATA TYPES:\n",
      "float64    35\n",
      "object      8\n",
      "int64       8\n",
      "Name: count, dtype: int64\n",
      "\n",
      "3. MISSING VALUES:\n",
      "                            Missing  Percentage\n",
      "all_task_dinhky                 168       37.00\n",
      "all_task_normal                  27        5.95\n",
      "num_tasks                        22        4.85\n",
      "num_cleaning_tasks               22        4.85\n",
      "num_trash_collection_tasks       22        4.85\n",
      "num_monitoring_tasks             22        4.85\n",
      "num_room_cleaning_tasks          22        4.85\n",
      "num_deep_cleaning_tasks          22        4.85\n",
      "num_maintenance_tasks            22        4.85\n",
      "num_support_tasks                22        4.85\n",
      "num_other_tasks                  22        4.85\n",
      "num_wc_tasks                     22        4.85\n",
      "num_hallway_tasks                22        4.85\n",
      "num_lobby_tasks                  22        4.85\n",
      "num_patient_room_tasks           22        4.85\n",
      "num_clinic_room_tasks            22        4.85\n",
      "num_surgery_room_tasks           22        4.85\n",
      "num_outdoor_tasks                22        4.85\n",
      "num_elevator_tasks               22        4.85\n",
      "num_office_tasks                 22        4.85\n",
      "num_technical_room_tasks         22        4.85\n",
      "cleaning_ratio                   22        4.85\n",
      "trash_collection_ratio           22        4.85\n",
      "monitoring_ratio                 22        4.85\n",
      "room_cleaning_ratio              22        4.85\n",
      "area_diversity                   22        4.85\n",
      "task_complexity_score            22        4.85\n",
      "\n",
      "4. TARGET VARIABLE (so_luong):\n",
      "count    454.000000\n",
      "mean       4.484581\n",
      "std        6.626822\n",
      "min        0.000000\n",
      "25%        1.000000\n",
      "50%        2.000000\n",
      "75%        5.000000\n",
      "max       64.000000\n",
      "Name: so_luong, dtype: float64\n",
      "   Zero values: 16\n",
      "   Skewness: 3.69\n",
      "   Kurtosis: 19.63\n"
     ]
    }
   ],
   "source": [
    "print(\"=\"*80)\n",
    "print(\"📊 EXPLORATORY DATA ANALYSIS\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "print(\"\\n1. DATA INFO:\")\n",
    "print(f\"   Shape: {df.shape}\")\n",
    "print(f\"   Memory: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB\")\n",
    "\n",
    "print(\"\\n2. DATA TYPES:\")\n",
    "print(df.dtypes.value_counts())\n",
    "\n",
    "print(\"\\n3. MISSING VALUES:\")\n",
    "missing = df.isnull().sum()\n",
    "missing_pct = (missing / len(df) * 100).round(2)\n",
    "missing_df = pd.DataFrame({\n",
    "    'Missing': missing[missing > 0],\n",
    "    'Percentage': missing_pct[missing > 0]\n",
    "}).sort_values('Missing', ascending=False)\n",
    "print(missing_df)\n",
    "\n",
    "print(\"\\n4. TARGET VARIABLE (so_luong):\")\n",
    "print(df['so_luong'].describe())\n",
    "print(f\"   Zero values: {(df['so_luong'] == 0).sum()}\")\n",
    "print(f\"   Skewness: {df['so_luong'].skew():.2f}\")\n",
    "print(f\"   Kurtosis: {df['so_luong'].kurt():.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dbbf735",
   "metadata": {},
   "source": [
    "# 3️⃣ Visualizations - Target Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "75af8fc3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 146204 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ Saved: plots/01_target_analysis.png\n"
     ]
    }
   ],
   "source": [
    "# Target Analysis Visualizations\n",
    "fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n",
    "\n",
    "# 1. Histogram\n",
    "axes[0, 0].hist(df['so_luong'], bins=30, edgecolor='black', alpha=0.7)\n",
    "axes[0, 0].axvline(df['so_luong'].mean(), color='red', linestyle='--', label=f'Mean={df[\"so_luong\"].mean():.2f}')\n",
    "axes[0, 0].axvline(df['so_luong'].median(), color='green', linestyle='--', label=f'Median={df[\"so_luong\"].median():.2f}')\n",
    "axes[0, 0].set_xlabel('Số lượng nhân sự')\n",
    "axes[0, 0].set_ylabel('Frequency')\n",
    "axes[0, 0].set_title('Phân bố số lượng nhân sự')\n",
    "axes[0, 0].legend()\n",
    "axes[0, 0].grid(alpha=0.3)\n",
    "\n",
    "# 2. Boxplot\n",
    "axes[0, 1].boxplot(df['so_luong'], vert=True)\n",
    "axes[0, 1].set_ylabel('Số lượng nhân sự')\n",
    "axes[0, 1].set_title('Boxplot - Phát hiện outliers')\n",
    "axes[0, 1].grid(alpha=0.3)\n",
    "\n",
    "# 3. Distribution by loai_ca\n",
    "df.boxplot(column='so_luong', by='loai_ca', ax=axes[1, 0])\n",
    "axes[1, 0].set_xlabel('Loại ca')\n",
    "axes[1, 0].set_ylabel('Số lượng nhân sự')\n",
    "axes[1, 0].set_title('Phân bố theo loại ca')\n",
    "plt.sca(axes[1, 0])\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "\n",
    "# 4. Top features correlation\n",
    "numeric_cols = df.select_dtypes(include=[np.number]).columns\n",
    "corr_with_target = df[numeric_cols].corr()['so_luong'].sort_values(ascending=False)\n",
    "top_10 = corr_with_target[1:11]\n",
    "axes[1, 1].barh(range(len(top_10)), top_10.values)\n",
    "axes[1, 1].set_yticks(range(len(top_10)))\n",
    "axes[1, 1].set_yticklabels(top_10.index)\n",
    "axes[1, 1].set_xlabel('Correlation with so_luong')\n",
    "axes[1, 1].set_title('Top 10 Features có correlation cao nhất')\n",
    "axes[1, 1].grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/01_target_analysis.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "print(\"✅ Saved: plots/01_target_analysis.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "546ce253",
   "metadata": {},
   "source": [
    "# 4️⃣ Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4f4fe933",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "🧹 DATA CLEANING\n",
      "================================================================================\n",
      "\n",
      "1. Handling missing values...\n",
      "   Filled num_tasks with 0\n",
      "   Filled num_cleaning_tasks with 0\n",
      "   Filled num_trash_collection_tasks with 0\n",
      "   Filled num_monitoring_tasks with 0\n",
      "   Filled num_room_cleaning_tasks with 0\n",
      "   Filled num_deep_cleaning_tasks with 0\n",
      "   Filled num_maintenance_tasks with 0\n",
      "   Filled num_support_tasks with 0\n",
      "   Filled num_other_tasks with 0\n",
      "   Filled num_wc_tasks with 0\n",
      "   Filled num_hallway_tasks with 0\n",
      "   Filled num_lobby_tasks with 0\n",
      "   Filled num_patient_room_tasks with 0\n",
      "   Filled num_clinic_room_tasks with 0\n",
      "   Filled num_surgery_room_tasks with 0\n",
      "   Filled num_outdoor_tasks with 0\n",
      "   Filled num_elevator_tasks with 0\n",
      "   Filled num_office_tasks with 0\n",
      "   Filled num_technical_room_tasks with 0\n",
      "   Filled cleaning_ratio with 0\n",
      "   Filled trash_collection_ratio with 0\n",
      "   Filled monitoring_ratio with 0\n",
      "   Filled room_cleaning_ratio with 0\n",
      "   Filled area_diversity with 0\n",
      "   Filled task_complexity_score with 0\n",
      "\n",
      "2. Analyzing target outliers...\n",
      "   Zero values: 16\n",
      "   Decision: KEEP all values (including zeros)\n",
      "\n",
      "3. Checking duplicates...\n",
      "   Duplicates found: 9\n",
      "\n",
      "4. Dropping unnecessary columns...\n",
      "   Dropped: ['ma_dia_diem', 'all_task_normal', 'all_task_dinhky', 'ten_toa_thap']\n",
      "\n",
      "5. Validation after cleaning:\n",
      "   Shape: (445, 47)\n",
      "   Missing values: 0\n",
      "\n",
      "✅ Saved: data/cleaned/CLEANED_DATA.csv\n"
     ]
    }
   ],
   "source": [
    "print(\"=\"*80)\n",
    "print(\"🧹 DATA CLEANING\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "df_clean = df.copy()\n",
    "\n",
    "# 1. Handle missing values\n",
    "print(\"\\n1. Handling missing values...\")\n",
    "\n",
    "# Task features: fill with 0\n",
    "task_features = [col for col in df_clean.columns if col.startswith('num_') or col.endswith('_ratio') or col in ['area_diversity', 'task_complexity_score']]\n",
    "for col in task_features:\n",
    "    if df_clean[col].isnull().sum() > 0:\n",
    "        df_clean[col] = df_clean[col].fillna(0)\n",
    "        print(f\"   Filled {col} with 0\")\n",
    "\n",
    "# Building features: fill with median\n",
    "building_features = [col for col in df_clean.columns if col.startswith('dien_tich_') or col in ['so_tang', 'so_cua_thang_may']]\n",
    "for col in building_features:\n",
    "    if df_clean[col].isnull().sum() > 0:\n",
    "        median_val = df_clean[col].median()\n",
    "        df_clean[col] = df_clean[col].fillna(median_val)\n",
    "        print(f\"   Filled {col} with median={median_val}\")\n",
    "\n",
    "# Categorical: fill with mode\n",
    "cat_features = ['loai_hinh', 'muc_do_luu_luong']\n",
    "for col in cat_features:\n",
    "    if col in df_clean.columns and df_clean[col].isnull().sum() > 0:\n",
    "        mode_val = df_clean[col].mode()[0]\n",
    "        df_clean[col] = df_clean[col].fillna(mode_val)\n",
    "        print(f\"   Filled {col} with mode={mode_val}\")\n",
    "\n",
    "# 2. Handle outliers (keep all for now)\n",
    "print(\"\\n2. Analyzing target outliers...\")\n",
    "print(f\"   Zero values: {(df_clean['so_luong'] == 0).sum()}\")\n",
    "print(\"   Decision: KEEP all values (including zeros)\")\n",
    "\n",
    "# 3. Remove duplicates\n",
    "print(\"\\n3. Checking duplicates...\")\n",
    "dups = df_clean.duplicated().sum()\n",
    "print(f\"   Duplicates found: {dups}\")\n",
    "if dups > 0:\n",
    "    df_clean = df_clean.drop_duplicates()\n",
    "\n",
    "# 4. Drop unnecessary columns\n",
    "print(\"\\n4. Dropping unnecessary columns...\")\n",
    "cols_to_drop = ['ma_dia_diem', 'all_task_normal', 'all_task_dinhky', 'ten_toa_thap']\n",
    "cols_to_drop = [col for col in cols_to_drop if col in df_clean.columns]\n",
    "df_clean = df_clean.drop(columns=cols_to_drop)\n",
    "print(f\"   Dropped: {cols_to_drop}\")\n",
    "\n",
    "# 5. Validate\n",
    "print(\"\\n5. Validation after cleaning:\")\n",
    "print(f\"   Shape: {df_clean.shape}\")\n",
    "print(f\"   Missing values: {df_clean.isnull().sum().sum()}\")\n",
    "\n",
    "# Save\n",
    "df_clean.to_csv('data/cleaned/CLEANED_DATA.csv', index=False)\n",
    "print(\"\\n✅ Saved: data/cleaned/CLEANED_DATA.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6468736",
   "metadata": {},
   "source": [
    "# 5️⃣ Feature Engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "4b5fe9a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "⚙️ FEATURE ENGINEERING\n",
      "================================================================================\n",
      "\n",
      "1. Creating time features...\n",
      "   ✅ Created time features\n",
      "\n",
      "2. Creating interaction features...\n",
      "   ✅ Created interaction features\n",
      "\n",
      "3. Creating aggregation features...\n",
      "   ✅ Created aggregation features\n",
      "\n",
      "4. Encoding categorical features...\n",
      "   ✅ One-hot encoded loai_ca\n",
      "   ✅ Label encoded loai_hinh\n",
      "   ✅ Label encoded muc_do_luu_luong\n",
      "\n",
      "✅ Feature engineering complete! Shape: (445, 64)\n",
      "✅ Saved: data/cleaned/ENGINEERED_DATA.csv\n"
     ]
    }
   ],
   "source": [
    "print(\"=\"*80)\n",
    "print(\"⚙️ FEATURE ENGINEERING\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "df_eng = df_clean.copy()\n",
    "\n",
    "# A. TIME FEATURES\n",
    "print(\"\\n1. Creating time features...\")\n",
    "\n",
    "def parse_time(time_val):\n",
    "    if pd.isna(time_val):\n",
    "        return 0\n",
    "    \n",
    "    # If it's already a datetime object\n",
    "    if isinstance(time_val, pd.Timestamp) or hasattr(time_val, 'hour'):\n",
    "        return time_val.hour\n",
    "    \n",
    "    # If it's a string\n",
    "    if isinstance(time_val, str):\n",
    "        time_val = time_val.strip()\n",
    "        \n",
    "        # Format: \"2025-01-01 22:00:00\" or \"2025-01-01 22\"\n",
    "        if ' ' in time_val:\n",
    "            # Split by space and get the time part\n",
    "            parts = time_val.split(' ')\n",
    "            if len(parts) >= 2:\n",
    "                time_part = parts[1]  # Get time part after space\n",
    "                if ':' in time_part:\n",
    "                    return int(time_part.split(':')[0])\n",
    "                else:\n",
    "                    return int(time_part)\n",
    "        \n",
    "        # Format: \"22:00:00\" or \"22:00\"\n",
    "        elif ':' in time_val:\n",
    "            return int(time_val.split(':')[0])\n",
    "        \n",
    "        # Try to parse as integer\n",
    "        try:\n",
    "            return int(float(time_val))\n",
    "        except:\n",
    "            return 0\n",
    "    \n",
    "    # If it's a number\n",
    "    if isinstance(time_val, (int, float)):\n",
    "        return int(time_val) if not np.isnan(time_val) else 0\n",
    "    \n",
    "    return 0\n",
    "\n",
    "df_eng['hour_start'] = df_eng['bat_dau'].apply(parse_time)\n",
    "df_eng['hour_end'] = df_eng['ket_thuc'].apply(parse_time)\n",
    "\n",
    "def parse_work_hours(time_str):\n",
    "    if pd.isna(time_str):\n",
    "        return 8.0\n",
    "    if isinstance(time_str, str) and ':' in time_str:\n",
    "        parts = time_str.split(':')\n",
    "        return float(parts[0]) + float(parts[1])/60 if len(parts) > 1 else float(parts[0])\n",
    "    return 8.0\n",
    "\n",
    "df_eng['work_hours_numeric'] = df_eng['tong_gio_lam'].apply(parse_work_hours)\n",
    "\n",
    "# Shift indicators\n",
    "df_eng['is_morning_shift'] = ((df_eng['hour_start'] >= 6) & (df_eng['hour_start'] < 12)).astype(int)\n",
    "df_eng['is_afternoon_shift'] = ((df_eng['hour_start'] >= 12) & (df_eng['hour_start'] < 18)).astype(int)\n",
    "df_eng['is_evening_shift'] = ((df_eng['hour_start'] >= 18) & (df_eng['hour_start'] < 24)).astype(int)\n",
    "df_eng['is_night_shift'] = ((df_eng['hour_start'] >= 0) & (df_eng['hour_start'] < 6)).astype(int)\n",
    "\n",
    "print(\"   ✅ Created time features\")\n",
    "\n",
    "# B. INTERACTION FEATURES\n",
    "print(\"\\n2. Creating interaction features...\")\n",
    "\n",
    "df_eng['tasks_per_hour'] = df_eng['num_tasks'] / (df_eng['work_hours_numeric'] + 0.1)\n",
    "df_eng['tasks_per_floor'] = df_eng['num_tasks'] / (df_eng['so_tang'] + 1)\n",
    "df_eng['wc_per_floor'] = df_eng['num_wc_tasks'] / (df_eng['so_tang'] + 1)\n",
    "df_eng['cleaning_workload'] = df_eng['num_cleaning_tasks'] * df_eng['area_diversity']\n",
    "\n",
    "print(\"   ✅ Created interaction features\")\n",
    "\n",
    "# C. AGGREGATION FEATURES\n",
    "print(\"\\n3. Creating aggregation features...\")\n",
    "\n",
    "area_cols = [col for col in df_eng.columns if col.startswith('dien_tich_')]\n",
    "df_eng['total_area'] = df_eng[area_cols].sum(axis=1)\n",
    "df_eng['area_per_floor'] = df_eng['total_area'] / (df_eng['so_tang'] + 1)\n",
    "\n",
    "special_cols = ['num_patient_room_tasks', 'num_surgery_room_tasks', 'num_clinic_room_tasks']\n",
    "df_eng['has_special_areas'] = (df_eng[special_cols].sum(axis=1) > 0).astype(int)\n",
    "\n",
    "print(\"   ✅ Created aggregation features\")\n",
    "\n",
    "# D. CATEGORICAL ENCODING\n",
    "print(\"\\n4. Encoding categorical features...\")\n",
    "\n",
    "# One-hot for loai_ca\n",
    "if 'loai_ca' in df_eng.columns:\n",
    "    loai_ca_dummies = pd.get_dummies(df_eng['loai_ca'], prefix='loai_ca', drop_first=True)\n",
    "    df_eng = pd.concat([df_eng, loai_ca_dummies], axis=1)\n",
    "    df_eng = df_eng.drop('loai_ca', axis=1)\n",
    "    print(f\"   ✅ One-hot encoded loai_ca\")\n",
    "\n",
    "# Label encoding for ordinal\n",
    "ordinal_map = {\n",
    "    'loai_hinh': {'Văn phòng': 0, 'Bệnh viện': 1, 'Trung tâm thương mại': 2, 'Khác': 3},\n",
    "    'muc_do_luu_luong': {'Thấp': 0, 'Trung bình': 1, 'Cao': 2}\n",
    "}\n",
    "\n",
    "for col, mapping in ordinal_map.items():\n",
    "    if col in df_eng.columns:\n",
    "        df_eng[col] = df_eng[col].map(mapping).fillna(0).astype(int)\n",
    "        print(f\"   ✅ Label encoded {col}\")\n",
    "\n",
    "# Drop original time columns\n",
    "time_cols = ['bat_dau', 'ket_thuc', 'tong_gio_lam']\n",
    "df_eng = df_eng.drop(columns=[c for c in time_cols if c in df_eng.columns])\n",
    "\n",
    "print(f\"\\n✅ Feature engineering complete! Shape: {df_eng.shape}\")\n",
    "\n",
    "# Save\n",
    "df_eng.to_csv('data/cleaned/ENGINEERED_DATA.csv', index=False)\n",
    "print(\"✅ Saved: data/cleaned/ENGINEERED_DATA.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e59fd2b8",
   "metadata": {},
   "source": [
    "# 6️⃣ Feature Insights & Correlation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "d4d265f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 412484 characters]",
      "text/plain": [
       "<Figure size 1200x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✅ Saved: plots/02_correlation_heatmap.png\n",
      "\n",
      "📊 Top 10 Correlated Features:\n",
      "dien_tich_kinh                0.445751\n",
      "total_area                    0.339654\n",
      "num_trash_collection_tasks    0.334920\n",
      "dien_tich_wc                  0.327664\n",
      "num_tasks                     0.308021\n",
      "tasks_per_hour                0.308021\n",
      "cleaning_workload             0.305543\n",
      "num_support_tasks             0.294081\n",
      "num_cleaning_tasks            0.291498\n",
      "task_complexity_score         0.271976\n",
      "Name: so_luong, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# Correlation Heatmap - Top 20 Features\n",
    "numeric_cols = df_eng.select_dtypes(include=[np.number]).columns\n",
    "correlations = df_eng[numeric_cols].corr()['so_luong'].abs().sort_values(ascending=False)\n",
    "top_features = correlations[1:21].index.tolist()\n",
    "\n",
    "plt.figure(figsize=(12, 10))\n",
    "corr_matrix = df_eng[top_features + ['so_luong']].corr()\n",
    "sns.heatmap(corr_matrix, annot=True, fmt='.2f', cmap='coolwarm', center=0, \n",
    "            square=True, linewidths=1, cbar_kws={\"shrink\": 0.8})\n",
    "plt.title('Correlation Heatmap - Top 20 Features vs Target', fontsize=14, fontweight='bold')\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/02_correlation_heatmap.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "print(\"✅ Saved: plots/02_correlation_heatmap.png\")\n",
    "print(f\"\\n📊 Top 10 Correlated Features:\")\n",
    "print(correlations[1:11])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3af98202",
   "metadata": {},
   "source": [
    "# 7️⃣ Train/Validation/Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "0796eb02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "✂️ TRAIN/VALIDATION/TEST SPLIT\n",
      "================================================================================\n",
      "\n",
      "📊 Dataset: 445 samples, 63 features\n",
      "\n",
      "✅ Split complete:\n",
      "   Train: 311 samples (69.9%)\n",
      "   Val:   67 samples (15.1%)\n",
      "   Test:  67 samples (15.1%)\n",
      "\n",
      "📊 Target distribution:\n",
      "   Train - Mean: 4.60, Std: 6.02\n",
      "   Val   - Mean: 4.87, Std: 9.81\n",
      "   Test  - Mean: 3.91, Std: 5.78\n",
      "\n",
      "✅ Saved splits to data/splits/\n"
     ]
    }
   ],
   "source": [
    "print(\"=\"*80)\n",
    "print(\"✂️ TRAIN/VALIDATION/TEST SPLIT\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "# Separate features and target\n",
    "X = df_eng.drop('so_luong', axis=1)\n",
    "y = df_eng['so_luong']\n",
    "\n",
    "print(f\"\\n📊 Dataset: {X.shape[0]} samples, {X.shape[1]} features\")\n",
    "\n",
    "# Split: 70% train, 15% val, 15% test\n",
    "X_temp, X_test, y_temp, y_test = train_test_split(X, y, test_size=0.15, random_state=RANDOM_STATE)\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_temp, y_temp, test_size=0.176, random_state=RANDOM_STATE)\n",
    "\n",
    "print(f\"\\n✅ Split complete:\")\n",
    "print(f\"   Train: {X_train.shape[0]} samples ({X_train.shape[0]/len(X)*100:.1f}%)\")\n",
    "print(f\"   Val:   {X_val.shape[0]} samples ({X_val.shape[0]/len(X)*100:.1f}%)\")\n",
    "print(f\"   Test:  {X_test.shape[0]} samples ({X_test.shape[0]/len(X)*100:.1f}%)\")\n",
    "\n",
    "print(f\"\\n📊 Target distribution:\")\n",
    "print(f\"   Train - Mean: {y_train.mean():.2f}, Std: {y_train.std():.2f}\")\n",
    "print(f\"   Val   - Mean: {y_val.mean():.2f}, Std: {y_val.std():.2f}\")\n",
    "print(f\"   Test  - Mean: {y_test.mean():.2f}, Std: {y_test.std():.2f}\")\n",
    "\n",
    "# Save splits\n",
    "pd.concat([X_train, y_train], axis=1).to_csv('data/splits/train.csv', index=False)\n",
    "pd.concat([X_val, y_val], axis=1).to_csv('data/splits/val.csv', index=False)\n",
    "pd.concat([X_test, y_test], axis=1).to_csv('data/splits/test.csv', index=False)\n",
    "print(\"\\n✅ Saved splits to data/splits/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "564c7690",
   "metadata": {},
   "source": [
    "# 8️⃣ Feature Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "08250ace",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "📏 FEATURE SCALING\n",
      "\n",
      "   Binary (no scaling): 12\n",
      "   One-hot (no scaling): 7\n",
      "   Numeric (will scale): 51\n",
      "\n",
      "✅ Scaling complete! Scaler saved to: models/scaler.pkl\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "print(\"📏 FEATURE SCALING\")\n",
    "\n",
    "# Identify columns to scale\n",
    "binary_cols = [col for col in X_train.columns if X_train[col].nunique() == 2 and set(X_train[col].unique()).issubset({0, 1})]\n",
    "onehot_cols = [col for col in X_train.columns if col.startswith('loai_ca_')]\n",
    "cols_to_scale = [col for col in X_train.columns if col not in binary_cols and col not in onehot_cols]\n",
    "\n",
    "print(f\"\\n   Binary (no scaling): {len(binary_cols)}\")\n",
    "print(f\"   One-hot (no scaling): {len(onehot_cols)}\")\n",
    "print(f\"   Numeric (will scale): {len(cols_to_scale)}\")\n",
    "\n",
    "# Scale\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = X_train.copy()\n",
    "X_val_scaled = X_val.copy()\n",
    "X_test_scaled = X_test.copy()\n",
    "\n",
    "X_train_scaled[cols_to_scale] = scaler.fit_transform(X_train[cols_to_scale])\n",
    "X_val_scaled[cols_to_scale] = scaler.transform(X_val[cols_to_scale])\n",
    "X_test_scaled[cols_to_scale] = scaler.transform(X_test[cols_to_scale])\n",
    "\n",
    "# Save scaler\n",
    "with open('models/scaler.pkl', 'wb') as f:\n",
    "    pickle.dump(scaler, f)\n",
    "\n",
    "print(\"\\n✅ Scaling complete! Scaler saved to: models/scaler.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a947162",
   "metadata": {},
   "source": [
    "# 9️⃣ Model Training & Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "cab9d093",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "🤖 MODEL TRAINING & EVALUATION\n",
      "================================================================================\n",
      "\n",
      "1️⃣ Training: Mean Baseline...\n",
      "   ✅ Val MAE: 5.108, R²: 0.000\n",
      "\n",
      "2️⃣ Training: Linear Regression...\n",
      "   ✅ Val MAE: 4.931, R²: -0.082\n",
      "\n",
      "3️⃣ Training: Decision Tree...\n",
      "   ✅ Val MAE: 3.384, R²: 0.414\n",
      "\n",
      "4️⃣ Training: Random Forest...\n",
      "   ✅ Val MAE: 3.557, R²: 0.309\n",
      "\n",
      "5️⃣ Training: Gradient Boosting...\n",
      "   ✅ Val MAE: 3.557, R²: 0.309\n",
      "\n",
      "5️⃣ Training: Gradient Boosting...\n",
      "   ✅ Val MAE: 3.036, R²: 0.398\n",
      "\n",
      "6️⃣ Training: XGBoost...\n",
      "   ✅ Val MAE: 3.260, R²: 0.323\n",
      "\n",
      "7️⃣ Training: LightGBM...\n",
      "   ✅ Val MAE: 3.603, R²: 0.295\n",
      "\n",
      "✅ All models trained!\n",
      "   ✅ Val MAE: 3.036, R²: 0.398\n",
      "\n",
      "6️⃣ Training: XGBoost...\n",
      "   ✅ Val MAE: 3.260, R²: 0.323\n",
      "\n",
      "7️⃣ Training: LightGBM...\n",
      "   ✅ Val MAE: 3.603, R²: 0.295\n",
      "\n",
      "✅ All models trained!\n"
     ]
    }
   ],
   "source": [
    "print(\"=\"*80)\n",
    "print(\"🤖 MODEL TRAINING & EVALUATION\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "# Evaluation function\n",
    "def evaluate_model(model, X_train, y_train, X_val, y_val, model_name=\"Model\"):\n",
    "    model.fit(X_train, y_train)\n",
    "    \n",
    "    y_train_pred = model.predict(X_train)\n",
    "    y_val_pred = model.predict(X_val)\n",
    "    \n",
    "    metrics = {\n",
    "        'Model': model_name,\n",
    "        'Train_MAE': mean_absolute_error(y_train, y_train_pred),\n",
    "        'Train_RMSE': np.sqrt(mean_squared_error(y_train, y_train_pred)),\n",
    "        'Train_R2': r2_score(y_train, y_train_pred),\n",
    "        'Val_MAE': mean_absolute_error(y_val, y_val_pred),\n",
    "        'Val_RMSE': np.sqrt(mean_squared_error(y_val, y_val_pred)),\n",
    "        'Val_R2': r2_score(y_val, y_val_pred),\n",
    "    }\n",
    "    \n",
    "    return metrics, model\n",
    "\n",
    "# Store results\n",
    "results = []\n",
    "models_dict = {}\n",
    "\n",
    "# 1. Mean Baseline\n",
    "print(\"\\n1️⃣ Training: Mean Baseline...\")\n",
    "mean_pred = np.full(len(y_val), y_train.mean())\n",
    "baseline_metrics = {\n",
    "    'Model': 'Mean Baseline',\n",
    "    'Train_MAE': mean_absolute_error(y_train, [y_train.mean()]*len(y_train)),\n",
    "    'Train_RMSE': np.sqrt(mean_squared_error(y_train, [y_train.mean()]*len(y_train))),\n",
    "    'Train_R2': 0.0,\n",
    "    'Val_MAE': mean_absolute_error(y_val, mean_pred),\n",
    "    'Val_RMSE': np.sqrt(mean_squared_error(y_val, mean_pred)),\n",
    "    'Val_R2': 0.0,\n",
    "}\n",
    "results.append(baseline_metrics)\n",
    "print(f\"   ✅ Val MAE: {baseline_metrics['Val_MAE']:.3f}, R²: {baseline_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "# 2. Linear Regression\n",
    "print(\"\\n2️⃣ Training: Linear Regression...\")\n",
    "lr = LinearRegression()\n",
    "lr_metrics, lr_model = evaluate_model(lr, X_train_scaled, y_train, X_val_scaled, y_val, \"Linear Regression\")\n",
    "results.append(lr_metrics)\n",
    "models_dict['Linear Regression'] = lr_model\n",
    "print(f\"   ✅ Val MAE: {lr_metrics['Val_MAE']:.3f}, R²: {lr_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "# 3. Decision Tree\n",
    "print(\"\\n3️⃣ Training: Decision Tree...\")\n",
    "dt = DecisionTreeRegressor(max_depth=10, min_samples_split=10, random_state=RANDOM_STATE)\n",
    "dt_metrics, dt_model = evaluate_model(dt, X_train_scaled, y_train, X_val_scaled, y_val, \"Decision Tree\")\n",
    "results.append(dt_metrics)\n",
    "models_dict['Decision Tree'] = dt_model\n",
    "print(f\"   ✅ Val MAE: {dt_metrics['Val_MAE']:.3f}, R²: {dt_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "# 4. Random Forest\n",
    "print(\"\\n4️⃣ Training: Random Forest...\")\n",
    "rf = RandomForestRegressor(n_estimators=100, max_depth=15, min_samples_split=5, \n",
    "                           random_state=RANDOM_STATE, n_jobs=-1)\n",
    "rf_metrics, rf_model = evaluate_model(rf, X_train_scaled, y_train, X_val_scaled, y_val, \"Random Forest\")\n",
    "results.append(rf_metrics)\n",
    "models_dict['Random Forest'] = rf_model\n",
    "print(f\"   ✅ Val MAE: {rf_metrics['Val_MAE']:.3f}, R²: {rf_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "# 5. Gradient Boosting\n",
    "print(\"\\n5️⃣ Training: Gradient Boosting...\")\n",
    "gb = GradientBoostingRegressor(n_estimators=100, learning_rate=0.1, max_depth=5, \n",
    "                               random_state=RANDOM_STATE)\n",
    "gb_metrics, gb_model = evaluate_model(gb, X_train_scaled, y_train, X_val_scaled, y_val, \"Gradient Boosting\")\n",
    "results.append(gb_metrics)\n",
    "models_dict['Gradient Boosting'] = gb_model\n",
    "print(f\"   ✅ Val MAE: {gb_metrics['Val_MAE']:.3f}, R²: {gb_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "# 6. XGBoost\n",
    "print(\"\\n6️⃣ Training: XGBoost...\")\n",
    "xgb_model = xgb.XGBRegressor(n_estimators=100, learning_rate=0.1, max_depth=5, \n",
    "                             random_state=RANDOM_STATE, n_jobs=-1)\n",
    "xgb_metrics, xgb_trained = evaluate_model(xgb_model, X_train_scaled, y_train, X_val_scaled, y_val, \"XGBoost\")\n",
    "results.append(xgb_metrics)\n",
    "models_dict['XGBoost'] = xgb_trained\n",
    "print(f\"   ✅ Val MAE: {xgb_metrics['Val_MAE']:.3f}, R²: {xgb_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "# 7. LightGBM\n",
    "print(\"\\n7️⃣ Training: LightGBM...\")\n",
    "lgb_model = lgb.LGBMRegressor(n_estimators=100, learning_rate=0.1, max_depth=5, \n",
    "                              num_leaves=31, random_state=RANDOM_STATE, n_jobs=-1, verbose=-1)\n",
    "lgb_metrics, lgb_trained = evaluate_model(lgb_model, X_train_scaled, y_train, X_val_scaled, y_val, \"LightGBM\")\n",
    "results.append(lgb_metrics)\n",
    "models_dict['LightGBM'] = lgb_trained\n",
    "print(f\"   ✅ Val MAE: {lgb_metrics['Val_MAE']:.3f}, R²: {lgb_metrics['Val_R2']:.3f}\")\n",
    "\n",
    "print(\"\\n✅ All models trained!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf0aac4e",
   "metadata": {},
   "source": [
    "# 🔟 Model Comparison & Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "fd373a20",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================================================================\n",
      "📊 MODEL COMPARISON RESULTS\n",
      "================================================================================\n",
      "\n",
      "            Model  Train_MAE  Train_RMSE  Train_R2  Val_MAE  Val_RMSE    Val_R2\n",
      "    Mean Baseline   3.956762    6.008718  0.000000 5.108173  9.743329  0.000000\n",
      "Linear Regression   2.848743    4.306219  0.486396 4.931250 10.131361 -0.082034\n",
      "    Decision Tree   1.416510    2.927375  0.762648 3.384117  7.458279  0.413616\n",
      "    Random Forest   1.518825    2.651412  0.805289 3.557119  8.094827  0.309251\n",
      "Gradient Boosting   0.500222    1.163144  0.962528 3.035648  7.556467  0.398074\n",
      "          XGBoost   0.588875    1.234300  0.957803 3.259926  8.016752  0.322511\n",
      "         LightGBM   1.527800    2.679199  0.801186 3.602681  8.178939  0.294821\n",
      "\n",
      "🏆 Best Model: Decision Tree\n",
      "        Model  Train_MAE  Train_RMSE  Train_R2  Val_MAE  Val_RMSE   Val_R2\n",
      "Decision Tree    1.41651    2.927375  0.762648 3.384117  7.458279 0.413616\n",
      "\n",
      "✅ Saved: models/model_comparison.csv\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 56140 characters]",
      "text/plain": [
       "<Figure size 1800x500 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "✅ Saved: plots/03_model_comparison.png\n"
     ]
    }
   ],
   "source": [
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"📊 MODEL COMPARISON RESULTS\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "results_df = pd.DataFrame(results)\n",
    "print(\"\\n\" + results_df.to_string(index=False))\n",
    "\n",
    "# Find best model\n",
    "best_model_name = results_df.loc[results_df['Val_R2'].idxmax(), 'Model']\n",
    "print(f\"\\n🏆 Best Model: {best_model_name}\")\n",
    "print(results_df[results_df['Model'] == best_model_name].to_string(index=False))\n",
    "\n",
    "# Save results\n",
    "results_df.to_csv('models/model_comparison.csv', index=False)\n",
    "print(\"\\n✅ Saved: models/model_comparison.csv\")\n",
    "\n",
    "# Visualization\n",
    "fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n",
    "\n",
    "# MAE\n",
    "axes[0].barh(results_df['Model'], results_df['Val_MAE'])\n",
    "axes[0].set_xlabel('MAE (lower is better)')\n",
    "axes[0].set_title('Validation MAE Comparison')\n",
    "axes[0].invert_yaxis()\n",
    "axes[0].grid(alpha=0.3)\n",
    "\n",
    "# RMSE\n",
    "axes[1].barh(results_df['Model'], results_df['Val_RMSE'])\n",
    "axes[1].set_xlabel('RMSE (lower is better)')\n",
    "axes[1].set_title('Validation RMSE Comparison')\n",
    "axes[1].invert_yaxis()\n",
    "axes[1].grid(alpha=0.3)\n",
    "\n",
    "# R²\n",
    "axes[2].barh(results_df['Model'], results_df['Val_R2'])\n",
    "axes[2].set_xlabel('R² Score (higher is better)')\n",
    "axes[2].set_title('Validation R² Comparison')\n",
    "axes[2].axvline(x=0.6, color='green', linestyle='--', label='Target (0.6)')\n",
    "axes[2].invert_yaxis()\n",
    "axes[2].legend()\n",
    "axes[2].grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/03_model_comparison.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "print(\"\\n✅ Saved: plots/03_model_comparison.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55287b1b",
   "metadata": {},
   "source": [
    "## 11. Phân tích Feature Importance\n",
    "\n",
    "Phân tích các features quan trọng nhất từ các tree-based models để hiểu yếu tố nào ảnh hưởng nhiều nhất đến số lượng nhân sự cần thiết."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "becc1b84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 245596 characters]",
      "text/plain": [
       "<Figure size 2000x1600 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================================================================\n",
      "TOP 20 FEATURES - RANDOM FOREST\n",
      "================================================================================\n",
      "                   feature  importance\n",
      "           num_other_tasks    0.082177\n",
      "                  hour_end    0.077996\n",
      "         num_support_tasks    0.069436\n",
      "        loai_ca_Hành chính    0.066470\n",
      "                total_area    0.053417\n",
      "num_trash_collection_tasks    0.049941\n",
      "              dien_tich_wc    0.039263\n",
      "          so_cua_thang_may    0.035331\n",
      "                hour_start    0.031520\n",
      "           loai_ca_Ca sáng    0.030894\n",
      "       dien_tich_hanh_lang    0.028037\n",
      "      dien_tich_ngoai_canh    0.026666\n",
      "          is_morning_shift    0.025059\n",
      "         cleaning_workload    0.024196\n",
      "     task_complexity_score    0.019776\n",
      "           num_lobby_tasks    0.018538\n",
      "     num_maintenance_tasks    0.017922\n",
      "                 num_tasks    0.017356\n",
      "            tasks_per_hour    0.015197\n",
      "            area_per_floor    0.014709\n",
      "\n",
      "================================================================================\n",
      "TOP 20 FEATURES - GRADIENT BOOSTING\n",
      "================================================================================\n",
      "                   feature  importance\n",
      "         num_support_tasks    0.168944\n",
      "           num_other_tasks    0.099070\n",
      "                  hour_end    0.090977\n",
      "              dien_tich_wc    0.076057\n",
      "        loai_ca_Hành chính    0.072796\n",
      "       dien_tich_hanh_lang    0.055343\n",
      "                hour_start    0.047728\n",
      "num_trash_collection_tasks    0.045451\n",
      "         cleaning_workload    0.033784\n",
      "          so_cua_thang_may    0.031193\n",
      "           loai_ca_Ca sáng    0.020162\n",
      "    trash_collection_ratio    0.017943\n",
      "      dien_tich_ngoai_canh    0.016354\n",
      "        is_afternoon_shift    0.015254\n",
      "                 num_tasks    0.013324\n",
      "              num_wc_tasks    0.012245\n",
      "          is_morning_shift    0.012009\n",
      "            dien_tich_kinh    0.011908\n",
      "           dien_tich_phong    0.011767\n",
      "                   so_tang    0.011093\n",
      "\n",
      "================================================================================\n",
      "TOP 20 FEATURES - XGBOOST\n",
      "================================================================================\n",
      "                   feature  importance\n",
      "         num_support_tasks    0.185853\n",
      "           num_other_tasks    0.139603\n",
      "num_trash_collection_tasks    0.087514\n",
      "        loai_ca_Hành chính    0.047658\n",
      "         cleaning_workload    0.041509\n",
      "       dien_tich_hanh_lang    0.036502\n",
      "     task_complexity_score    0.035533\n",
      "                   so_tang    0.029777\n",
      "            cleaning_ratio    0.023369\n",
      "              dien_tich_wc    0.022166\n",
      "           num_lobby_tasks    0.020765\n",
      "              num_wc_tasks    0.020499\n",
      "         num_hallway_tasks    0.019875\n",
      "                  hour_end    0.019466\n",
      "             op_chan_tuong    0.018002\n",
      "     num_maintenance_tasks    0.016238\n",
      "    trash_collection_ratio    0.012965\n",
      "          so_cua_thang_may    0.012714\n",
      "          num_office_tasks    0.012486\n",
      "      dien_tich_ngoai_canh    0.012133\n",
      "\n",
      "================================================================================\n",
      "TOP 20 FEATURES - LIGHTGBM\n",
      "================================================================================\n",
      "                   feature  importance\n",
      "                  hour_end          90\n",
      "              dien_tich_wc          57\n",
      "         num_support_tasks          47\n",
      "        loai_ca_Hành chính          33\n",
      "                hour_start          32\n",
      "       dien_tich_hanh_lang          31\n",
      "                   so_tang          28\n",
      "    trash_collection_ratio          28\n",
      "           num_other_tasks          27\n",
      "num_trash_collection_tasks          24\n",
      "          so_cua_thang_may          24\n",
      "           loai_ca_Ca sáng          23\n",
      "           dien_tich_phong          21\n",
      "                   doc_ham          20\n",
      "                total_area          20\n",
      "            area_per_floor          19\n",
      "            cleaning_ratio          18\n",
      "                 num_tasks          17\n",
      "            dien_tich_kinh          17\n",
      "              wc_per_floor          17\n",
      "\n",
      "================================================================================\n",
      "TOP 20 FEATURES - TRUNG BÌNH TỪ TẤT CẢ MODELS\n",
      "================================================================================\n",
      "                   feature  mean_importance\n",
      "                  hour_end        22.547110\n",
      "              dien_tich_wc        14.284371\n",
      "         num_support_tasks        11.856058\n",
      "        loai_ca_Hành chính         8.296731\n",
      "                hour_start         8.022267\n",
      "       dien_tich_hanh_lang         7.779970\n",
      "                   so_tang         7.012657\n",
      "    trash_collection_ratio         7.010491\n",
      "           num_other_tasks         6.830212\n",
      "num_trash_collection_tasks         6.045726\n",
      "          so_cua_thang_may         6.019809\n",
      "           loai_ca_Ca sáng         5.764881\n",
      "           dien_tich_phong         5.256978\n",
      "                total_area         5.018384\n",
      "                   doc_ham         5.004036\n",
      "            area_per_floor         4.757241\n",
      "            cleaning_ratio         4.511669\n",
      "                 num_tasks         4.260054\n",
      "            dien_tich_kinh         4.258079\n",
      "              wc_per_floor         4.252550\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 72240 characters]",
      "text/plain": [
       "<Figure size 1200x800 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Lấy feature importance từ các tree-based models\n",
    "feature_names = X_train.columns\n",
    "\n",
    "# Random Forest Feature Importance\n",
    "rf_importance = pd.DataFrame({\n",
    "    'feature': feature_names,\n",
    "    'importance': models_dict['Random Forest'].feature_importances_\n",
    "}).sort_values('importance', ascending=False)\n",
    "\n",
    "# Gradient Boosting Feature Importance\n",
    "gb_importance = pd.DataFrame({\n",
    "    'feature': feature_names,\n",
    "    'importance': models_dict['Gradient Boosting'].feature_importances_\n",
    "}).sort_values('importance', ascending=False)\n",
    "\n",
    "# XGBoost Feature Importance\n",
    "xgb_importance = pd.DataFrame({\n",
    "    'feature': feature_names,\n",
    "    'importance': models_dict['XGBoost'].feature_importances_\n",
    "}).sort_values('importance', ascending=False)\n",
    "\n",
    "# LightGBM Feature Importance\n",
    "lgbm_importance = pd.DataFrame({\n",
    "    'feature': feature_names,\n",
    "    'importance': models_dict['LightGBM'].feature_importances_\n",
    "}).sort_values('importance', ascending=False)\n",
    "\n",
    "# Visualize Top 20 Features\n",
    "fig, axes = plt.subplots(2, 2, figsize=(20, 16))\n",
    "fig.suptitle('Feature Importance Comparison - Top 20 Features', fontsize=16)\n",
    "\n",
    "# Plot RF\n",
    "axes[0, 0].barh(range(20), rf_importance.head(20)['importance'])\n",
    "axes[0, 0].set_yticks(range(20))\n",
    "axes[0, 0].set_yticklabels(rf_importance.head(20)['feature'])\n",
    "axes[0, 0].set_xlabel('Importance')\n",
    "axes[0, 0].set_title('Random Forest')\n",
    "axes[0, 0].invert_yaxis()\n",
    "\n",
    "# Plot GB\n",
    "axes[0, 1].barh(range(20), gb_importance.head(20)['importance'])\n",
    "axes[0, 1].set_yticks(range(20))\n",
    "axes[0, 1].set_yticklabels(gb_importance.head(20)['feature'])\n",
    "axes[0, 1].set_xlabel('Importance')\n",
    "axes[0, 1].set_title('Gradient Boosting')\n",
    "axes[0, 1].invert_yaxis()\n",
    "\n",
    "# Plot XGB\n",
    "axes[1, 0].barh(range(20), xgb_importance.head(20)['importance'])\n",
    "axes[1, 0].set_yticks(range(20))\n",
    "axes[1, 0].set_yticklabels(xgb_importance.head(20)['feature'])\n",
    "axes[1, 0].set_xlabel('Importance')\n",
    "axes[1, 0].set_title('XGBoost')\n",
    "axes[1, 0].invert_yaxis()\n",
    "\n",
    "# Plot LGBM\n",
    "axes[1, 1].barh(range(20), lgbm_importance.head(20)['importance'])\n",
    "axes[1, 1].set_yticks(range(20))\n",
    "axes[1, 1].set_yticklabels(lgbm_importance.head(20)['feature'])\n",
    "axes[1, 1].set_xlabel('Importance')\n",
    "axes[1, 1].set_title('LightGBM')\n",
    "axes[1, 1].invert_yaxis()\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/feature_importance_comparison.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "# Tổng hợp Top 20 Features chung\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"TOP 20 FEATURES - RANDOM FOREST\")\n",
    "print(\"=\"*80)\n",
    "print(rf_importance.head(20).to_string(index=False))\n",
    "\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"TOP 20 FEATURES - GRADIENT BOOSTING\")\n",
    "print(\"=\"*80)\n",
    "print(gb_importance.head(20).to_string(index=False))\n",
    "\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"TOP 20 FEATURES - XGBOOST\")\n",
    "print(\"=\"*80)\n",
    "print(xgb_importance.head(20).to_string(index=False))\n",
    "\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"TOP 20 FEATURES - LIGHTGBM\")\n",
    "print(\"=\"*80)\n",
    "print(lgbm_importance.head(20).to_string(index=False))\n",
    "\n",
    "# Tính trung bình importance từ tất cả các models\n",
    "all_importance = pd.DataFrame({\n",
    "    'feature': feature_names,\n",
    "    'rf_imp': models_dict['Random Forest'].feature_importances_,\n",
    "    'gb_imp': models_dict['Gradient Boosting'].feature_importances_,\n",
    "    'xgb_imp': models_dict['XGBoost'].feature_importances_,\n",
    "    'lgbm_imp': models_dict['LightGBM'].feature_importances_\n",
    "})\n",
    "\n",
    "all_importance['mean_importance'] = all_importance[['rf_imp', 'gb_imp', 'xgb_imp', 'lgbm_imp']].mean(axis=1)\n",
    "all_importance = all_importance.sort_values('mean_importance', ascending=False)\n",
    "\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"TOP 20 FEATURES - TRUNG BÌNH TỪ TẤT CẢ MODELS\")\n",
    "print(\"=\"*80)\n",
    "print(all_importance[['feature', 'mean_importance']].head(20).to_string(index=False))\n",
    "\n",
    "# Visualize average importance\n",
    "plt.figure(figsize=(12, 8))\n",
    "plt.barh(range(20), all_importance.head(20)['mean_importance'])\n",
    "plt.yticks(range(20), all_importance.head(20)['feature'])\n",
    "plt.xlabel('Average Importance', fontsize=12)\n",
    "plt.title('Top 20 Features - Average Importance Across All Models', fontsize=14)\n",
    "plt.gca().invert_yaxis()\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/average_feature_importance.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9691c38",
   "metadata": {},
   "source": [
    "## 12. Đánh Giá Chi Tiết Trên Test Set\n",
    "\n",
    "Đánh giá hiệu suất của best model trên test set và phân tích các trường hợp dự đoán sai."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e0e2e443",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================================================\n",
      "BEST MODEL: Gradient Boosting\n",
      "================================================================================\n",
      "\n",
      "Test Set Performance:\n",
      "  MAE:  2.0764\n",
      "  MSE:  11.8555\n",
      "  RMSE: 3.4432\n",
      "  R²:   0.6402\n",
      "\n",
      "Comparison with Validation Set:\n",
      "  Validation MAE: 3.0356 | Test MAE: 2.0764 | Diff: 0.9592\n",
      "  Validation R²:  0.3981 | Test R²:  0.6402 | Diff: 0.2421\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 104444 characters]",
      "text/plain": [
       "<Figure size 2000x600 with 3 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================================================================\n",
      "TOP 20 TRƯỜNG HỢP DỰ ĐOÁN SAI NHẤT\n",
      "================================================================================\n",
      "     actual  predicted      error  abs_error\n",
      "133      16   3.233567  12.766433  12.766433\n",
      "31        5  16.012736 -11.012736  11.012736\n",
      "57        0   8.738258  -8.738258   8.738258\n",
      "15       16   7.741727   8.258273   8.258273\n",
      "56       26  18.073003   7.926997   7.926997\n",
      "11       29  21.558546   7.441454   7.441454\n",
      "76       13   7.052550   5.947450   5.947450\n",
      "245      10   4.987020   5.012980   5.012980\n",
      "441       1   5.795453  -4.795453   4.795453\n",
      "369      20  15.401711   4.598289   4.598289\n",
      "149       1   5.097772  -4.097772   4.097772\n",
      "397       4   7.956151  -3.956151   3.956151\n",
      "9         1   4.872358  -3.872358   3.872358\n",
      "366       2   5.717268  -3.717268   3.717268\n",
      "285       2   5.616980  -3.616980   3.616980\n",
      "430       1   4.032095  -3.032095   3.032095\n",
      "79       11   8.008475   2.991525   2.991525\n",
      "91        3   5.866139  -2.866139   2.866139\n",
      "77        2   4.692245  -2.692245   2.692245\n",
      "447       1   2.888733  -1.888733   1.888733\n",
      "\n",
      "================================================================================\n",
      "PHÂN TÍCH LỖI\n",
      "================================================================================\n",
      "Mean Absolute Error: 2.0764\n",
      "Median Absolute Error: 0.9094\n",
      "Max Absolute Error: 12.7664\n",
      "Min Absolute Error: 0.0063\n",
      "Std of Absolute Error: 2.7674\n",
      "\n",
      "Phân vị lỗi tuyệt đối:\n",
      "  25th percentile: 0.2435\n",
      "  50th percentile: 0.9094\n",
      "  75th percentile: 2.9288\n",
      "  90th percentile: 5.3868\n",
      "  95th percentile: 8.1589\n",
      "\n",
      "✓ Kết quả test set đã được lưu vào 'data/test_set_predictions.csv'\n"
     ]
    }
   ],
   "source": [
    "# Chọn best model (theo MAE thấp nhất)\n",
    "best_model_name = results_df.loc[results_df['Val_MAE'].idxmin(), 'Model']\n",
    "best_model = models_dict[best_model_name]\n",
    "\n",
    "print(\"=\"*80)\n",
    "print(f\"BEST MODEL: {best_model_name}\")\n",
    "print(\"=\"*80)\n",
    "\n",
    "# Dự đoán trên test set\n",
    "y_test_pred = best_model.predict(X_test_scaled)\n",
    "\n",
    "# Tính toán metrics trên test set\n",
    "test_mae = mean_absolute_error(y_test, y_test_pred)\n",
    "test_mse = mean_squared_error(y_test, y_test_pred)\n",
    "test_rmse = np.sqrt(test_mse)\n",
    "test_r2 = r2_score(y_test, y_test_pred)\n",
    "\n",
    "print(f\"\\nTest Set Performance:\")\n",
    "print(f\"  MAE:  {test_mae:.4f}\")\n",
    "print(f\"  MSE:  {test_mse:.4f}\")\n",
    "print(f\"  RMSE: {test_rmse:.4f}\")\n",
    "print(f\"  R²:   {test_r2:.4f}\")\n",
    "\n",
    "# So sánh với validation set\n",
    "val_mae = results_df.loc[results_df['Model'] == best_model_name, 'Val_MAE'].values[0]\n",
    "val_r2 = results_df.loc[results_df['Model'] == best_model_name, 'Val_R2'].values[0]\n",
    "\n",
    "print(f\"\\nComparison with Validation Set:\")\n",
    "print(f\"  Validation MAE: {val_mae:.4f} | Test MAE: {test_mae:.4f} | Diff: {abs(test_mae - val_mae):.4f}\")\n",
    "print(f\"  Validation R²:  {val_r2:.4f} | Test R²:  {test_r2:.4f} | Diff: {abs(test_r2 - val_r2):.4f}\")\n",
    "\n",
    "# Visualize predictions vs actual\n",
    "fig, axes = plt.subplots(1, 3, figsize=(20, 6))\n",
    "\n",
    "# Scatter plot\n",
    "axes[0].scatter(y_test, y_test_pred, alpha=0.5, edgecolors='k')\n",
    "axes[0].plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], 'r--', lw=2)\n",
    "axes[0].set_xlabel('Actual', fontsize=12)\n",
    "axes[0].set_ylabel('Predicted', fontsize=12)\n",
    "axes[0].set_title(f'{best_model_name} - Test Set\\nPredicted vs Actual', fontsize=14)\n",
    "axes[0].grid(True, alpha=0.3)\n",
    "\n",
    "# Residuals plot\n",
    "residuals = y_test - y_test_pred\n",
    "axes[1].scatter(y_test_pred, residuals, alpha=0.5, edgecolors='k')\n",
    "axes[1].axhline(y=0, color='r', linestyle='--', lw=2)\n",
    "axes[1].set_xlabel('Predicted', fontsize=12)\n",
    "axes[1].set_ylabel('Residuals (Actual - Predicted)', fontsize=12)\n",
    "axes[1].set_title('Residual Plot', fontsize=14)\n",
    "axes[1].grid(True, alpha=0.3)\n",
    "\n",
    "# Residuals distribution\n",
    "axes[2].hist(residuals, bins=50, edgecolor='black', alpha=0.7)\n",
    "axes[2].axvline(x=0, color='r', linestyle='--', lw=2)\n",
    "axes[2].set_xlabel('Residuals', fontsize=12)\n",
    "axes[2].set_ylabel('Frequency', fontsize=12)\n",
    "axes[2].set_title('Distribution of Residuals', fontsize=14)\n",
    "axes[2].grid(True, alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig('plots/test_set_evaluation.png', dpi=300, bbox_inches='tight')\n",
    "plt.show()\n",
    "\n",
    "# Phân tích các trường hợp dự đoán sai nhất\n",
    "test_results = pd.DataFrame({\n",
    "    'actual': y_test,\n",
    "    'predicted': y_test_pred,\n",
    "    'error': residuals,\n",
    "    'abs_error': np.abs(residuals)\n",
    "})\n",
    "test_results = test_results.sort_values('abs_error', ascending=False)\n",
    "\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"TOP 20 TRƯỜNG HỢP DỰ ĐOÁN SAI NHẤT\")\n",
    "print(\"=\"*80)\n",
    "print(test_results.head(20))\n",
    "\n",
    "print(\"\\n\" + \"=\"*80)\n",
    "print(\"PHÂN TÍCH LỖI\")\n",
    "print(\"=\"*80)\n",
    "print(f\"Mean Absolute Error: {test_mae:.4f}\")\n",
    "print(f\"Median Absolute Error: {np.median(test_results['abs_error']):.4f}\")\n",
    "print(f\"Max Absolute Error: {test_results['abs_error'].max():.4f}\")\n",
    "print(f\"Min Absolute Error: {test_results['abs_error'].min():.4f}\")\n",
    "print(f\"Std of Absolute Error: {test_results['abs_error'].std():.4f}\")\n",
    "\n",
    "# Phân tích lỗi theo phân vị\n",
    "print(f\"\\nPhân vị lỗi tuyệt đối:\")\n",
    "print(f\"  25th percentile: {test_results['abs_error'].quantile(0.25):.4f}\")\n",
    "print(f\"  50th percentile: {test_results['abs_error'].quantile(0.50):.4f}\")\n",
    "print(f\"  75th percentile: {test_results['abs_error'].quantile(0.75):.4f}\")\n",
    "print(f\"  90th percentile: {test_results['abs_error'].quantile(0.90):.4f}\")\n",
    "print(f\"  95th percentile: {test_results['abs_error'].quantile(0.95):.4f}\")\n",
    "\n",
    "# Lưu kết quả test set\n",
    "test_results.to_csv('data/test_set_predictions.csv', index=False)\n",
    "print(\"\\n✓ Kết quả test set đã được lưu vào 'data/test_set_predictions.csv'\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
