nguyenletai16122001
/
predict_caLamviec_nhansu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
predict_caLamviec_nhansu/final.ipynb

3713 lines
136 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "code",
"execution_count": 12,
"id": "76aa1b75",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Loaded: final_2.xlsx | sheet: final\n",
"Shape (raw): (401, 42)\n",
"\n",
"=== TARGET SUMMARY (so_luong) ===\n",
"count 401.000000\n",
"mean 4.660848\n",
"std 6.799242\n",
"min 0.000000\n",
"25% 1.000000\n",
"50% 2.000000\n",
"75% 5.000000\n",
"max 64.000000\n",
"Name: so_luong, dtype: float64\n",
"Missing target: 0\n",
"Negative target: 0\n",
"Zero target: 3\n",
"\n",
"Duplicate full rows: 7\n",
"Shape (dedup): (394, 42)\n",
"\n",
"Columns: 42\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>ma_dia_diem</th>\n",
" <th>all_task_normal</th>\n",
" <th>all_task_dinhky</th>\n",
" <th>loai_ca</th>\n",
" <th>bat_dau</th>\n",
" <th>ket_thuc</th>\n",
" <th>tong_gio_lam</th>\n",
" <th>so_ca_cua_toa</th>\n",
" <th>so_luong</th>\n",
" <th>num_tasks</th>\n",
" <th>...</th>\n",
" <th>dien_tich_tham</th>\n",
" <th>doc_ham</th>\n",
" <th>vien_phan_quang</th>\n",
" <th>op_tuong</th>\n",
" <th>op_chan_tuong</th>\n",
" <th>ranh_thoat_nuoc</th>\n",
" <th>dien_tich_kinh</th>\n",
" <th>num_medical_tasks_total</th>\n",
" <th>num_indoor_room_tasks</th>\n",
" <th>is_tasks_text_missing</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>115-2</td>\n",
" <td>Làm sạch toàn bộ phòng giao dịch tầng 1 (kể cả...</td>\n",
" <td>NaN</td>\n",
" <td>Part time</td>\n",
" <td>06:30:00</td>\n",
" <td>10:30:00</td>\n",
" <td>4.0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>7</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>20.0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>101-1</td>\n",
" <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
" <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
" <td>Hành chính</td>\n",
" <td>06:30:00</td>\n",
" <td>16:00:00</td>\n",
" <td>7.5</td>\n",
" <td>6</td>\n",
" <td>24</td>\n",
" <td>441</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>70</td>\n",
" <td>0</td>\n",
" <td>9176.0</td>\n",
" <td>89.0</td>\n",
" <td>25</td>\n",
" <td>894.0</td>\n",
" <td>112</td>\n",
" <td>39</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>101-1</td>\n",
" <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
" <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
" <td>Ca sáng</td>\n",
" <td>06:00:00</td>\n",
" <td>14:00:00</td>\n",
" <td>8.0</td>\n",
" <td>6</td>\n",
" <td>3</td>\n",
" <td>441</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>70</td>\n",
" <td>0</td>\n",
" <td>9176.0</td>\n",
" <td>89.0</td>\n",
" <td>25</td>\n",
" <td>894.0</td>\n",
" <td>112</td>\n",
" <td>39</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3 rows × 42 columns</p>\n",
"</div>"
],
"text/plain": [
" ma_dia_diem all_task_normal \\\n",
"0 115-2 Làm sạch toàn bộ phòng giao dịch tầng 1 (kể cả... \n",
"1 101-1 Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,... \n",
"2 101-1 Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,... \n",
"\n",
" all_task_dinhky loai_ca bat_dau \\\n",
"0 NaN Part time 06:30:00 \n",
"1 Lau bảng biển, bình cứu hỏa , cây nước hành la... Hành chính 06:30:00 \n",
"2 Lau bảng biển, bình cứu hỏa , cây nước hành la... Ca sáng 06:00:00 \n",
"\n",
" ket_thuc tong_gio_lam so_ca_cua_toa so_luong num_tasks ... \\\n",
"0 10:30:00 4.0 1 1 7 ... \n",
"1 16:00:00 7.5 6 24 441 ... \n",
"2 14:00:00 8.0 6 3 441 ... \n",
"\n",
" dien_tich_tham doc_ham vien_phan_quang op_tuong op_chan_tuong \\\n",
"0 0.0 0 0 0.0 0.0 \n",
"1 0.0 70 0 9176.0 89.0 \n",
"2 0.0 70 0 9176.0 89.0 \n",
"\n",
" ranh_thoat_nuoc dien_tich_kinh num_medical_tasks_total \\\n",
"0 0 20.0 0 \n",
"1 25 894.0 112 \n",
"2 25 894.0 112 \n",
"\n",
" num_indoor_room_tasks is_tasks_text_missing \n",
"0 1 0 \n",
"1 39 0 \n",
"2 39 0 \n",
"\n",
"[3 rows x 42 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"DATA_PATH = \"final_2.xlsx\"\n",
"SHEET = \"final\"\n",
"TARGET = \"so_luong\"\n",
"\n",
"# 1) Load\n",
"df = pd.read_excel(DATA_PATH, sheet_name=SHEET)\n",
"print(\"✅ Loaded:\", DATA_PATH, \"| sheet:\", SHEET)\n",
"print(\"Shape (raw):\", df.shape)\n",
"\n",
"# 2) Target sanity\n",
"assert TARGET in df.columns, f\"❌ Missing target column: {TARGET}\"\n",
"df[TARGET] = pd.to_numeric(df[TARGET], errors=\"coerce\")\n",
"\n",
"print(\"\\n=== TARGET SUMMARY (so_luong) ===\")\n",
"print(df[TARGET].describe())\n",
"print(\"Missing target:\", df[TARGET].isna().sum())\n",
"print(\"Negative target:\", (df[TARGET] < 0).sum())\n",
"print(\"Zero target:\", (df[TARGET] == 0).sum())\n",
"\n",
"# 3) Deduplicate full rows\n",
"dup = df.duplicated().sum()\n",
"print(\"\\nDuplicate full rows:\", dup)\n",
"if dup > 0:\n",
" df = df.drop_duplicates().reset_index(drop=True)\n",
"print(\"Shape (dedup):\", df.shape)\n",
"\n",
"# 4) Quick preview\n",
"print(\"\\nColumns:\", len(df.columns))\n",
"display(df.head(3))\n"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "421b7556",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Index(['ma_dia_diem', 'all_task_normal', 'all_task_dinhky', 'loai_ca',\n",
" 'bat_dau', 'ket_thuc', 'tong_gio_lam', 'so_ca_cua_toa', 'so_luong',\n",
" 'num_tasks', 'num_cleaning_tasks', 'num_trash_collection_tasks',\n",
" 'num_monitoring_tasks', 'num_deep_cleaning_tasks', 'num_support_tasks',\n",
" 'num_other_tasks', 'num_wc_tasks', 'num_hallway_tasks',\n",
" 'num_lobby_tasks', 'num_outdoor_tasks', 'num_elevator_tasks',\n",
" 'cleaning_ratio', 'trash_collection_ratio', 'monitoring_ratio',\n",
" 'area_diversity', 'so_tang', 'so_cua_thang_may', 'dien_tich_ngoai_canh',\n",
" 'dien_tich_sanh', 'dien_tich_hanh_lang', 'dien_tich_wc',\n",
" 'dien_tich_phong', 'dien_tich_tham', 'doc_ham', 'vien_phan_quang',\n",
" 'op_tuong', 'op_chan_tuong', 'ranh_thoat_nuoc', 'dien_tich_kinh',\n",
" 'num_medical_tasks_total', 'num_indoor_room_tasks',\n",
" 'is_tasks_text_missing'],\n",
" dtype='object')"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.columns"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "daf5a333",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ X shape: (394, 48) | y shape: (394,) | #buildings: 192\n",
"Columns sample: ['tong_gio_lam', 'so_ca_cua_toa', 'num_tasks', 'num_cleaning_tasks', 'num_trash_collection_tasks', 'num_monitoring_tasks', 'num_deep_cleaning_tasks', 'num_support_tasks', 'num_other_tasks', 'num_wc_tasks', 'num_hallway_tasks', 'num_lobby_tasks', 'num_outdoor_tasks', 'num_elevator_tasks', 'cleaning_ratio']\n",
"Any NaN left?: False\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>tong_gio_lam</th>\n",
" <th>so_ca_cua_toa</th>\n",
" <th>num_tasks</th>\n",
" <th>num_cleaning_tasks</th>\n",
" <th>num_trash_collection_tasks</th>\n",
" <th>num_monitoring_tasks</th>\n",
" <th>num_deep_cleaning_tasks</th>\n",
" <th>num_support_tasks</th>\n",
" <th>num_other_tasks</th>\n",
" <th>num_wc_tasks</th>\n",
" <th>...</th>\n",
" <th>is_night_shift</th>\n",
" <th>is_morning_shift</th>\n",
" <th>is_afternoon_shift</th>\n",
" <th>is_evening_shift</th>\n",
" <th>loai_ca_Ca chiều</th>\n",
" <th>loai_ca_Ca gãy</th>\n",
" <th>loai_ca_Ca sáng</th>\n",
" <th>loai_ca_Ca đêm</th>\n",
" <th>loai_ca_Hành chính</th>\n",
" <th>loai_ca_Part time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4.0</td>\n",
" <td>1</td>\n",
" <td>7</td>\n",
" <td>7</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>4</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>7.5</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>8.0</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>8.0</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>8.0</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>7.5</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>9.5</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>9.5</td>\n",
" <td>3</td>\n",
" <td>135</td>\n",
" <td>81</td>\n",
" <td>35</td>\n",
" <td>38</td>\n",
" <td>10</td>\n",
" <td>20</td>\n",
" <td>21</td>\n",
" <td>25</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>7.5</td>\n",
" <td>3</td>\n",
" <td>135</td>\n",
" <td>81</td>\n",
" <td>35</td>\n",
" <td>38</td>\n",
" <td>10</td>\n",
" <td>20</td>\n",
" <td>21</td>\n",
" <td>25</td>\n",
" <td>...</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>7.5</td>\n",
" <td>3</td>\n",
" <td>135</td>\n",
" <td>81</td>\n",
" <td>35</td>\n",
" <td>38</td>\n",
" <td>10</td>\n",
" <td>20</td>\n",
" <td>21</td>\n",
" <td>25</td>\n",
" <td>...</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>10 rows × 48 columns</p>\n",
"</div>"
],
"text/plain": [
" tong_gio_lam so_ca_cua_toa num_tasks num_cleaning_tasks \\\n",
"0 4.0 1 7 7 \n",
"1 7.5 6 441 258 \n",
"2 8.0 6 441 258 \n",
"3 8.0 6 441 258 \n",
"4 8.0 6 441 258 \n",
"5 7.5 6 441 258 \n",
"6 9.5 6 441 258 \n",
"7 9.5 3 135 81 \n",
"8 7.5 3 135 81 \n",
"9 7.5 3 135 81 \n",
"\n",
" num_trash_collection_tasks num_monitoring_tasks num_deep_cleaning_tasks \\\n",
"0 1 2 1 \n",
"1 145 134 75 \n",
"2 145 134 75 \n",
"3 145 134 75 \n",
"4 145 134 75 \n",
"5 145 134 75 \n",
"6 145 134 75 \n",
"7 35 38 10 \n",
"8 35 38 10 \n",
"9 35 38 10 \n",
"\n",
" num_support_tasks num_other_tasks num_wc_tasks ... is_night_shift \\\n",
"0 0 0 4 ... 0 \n",
"1 57 45 89 ... 0 \n",
"2 57 45 89 ... 0 \n",
"3 57 45 89 ... 0 \n",
"4 57 45 89 ... 1 \n",
"5 57 45 89 ... 0 \n",
"6 57 45 89 ... 0 \n",
"7 20 21 25 ... 0 \n",
"8 20 21 25 ... 0 \n",
"9 20 21 25 ... 1 \n",
"\n",
" is_morning_shift is_afternoon_shift is_evening_shift loai_ca_Ca chiều \\\n",
"0 1 0 0 False \n",
"1 1 0 0 False \n",
"2 1 0 0 False \n",
"3 0 1 0 True \n",
"4 0 0 0 False \n",
"5 0 1 0 False \n",
"6 1 0 0 False \n",
"7 1 0 0 False \n",
"8 1 0 0 False \n",
"9 0 0 0 False \n",
"\n",
" loai_ca_Ca gãy loai_ca_Ca sáng loai_ca_Ca đêm loai_ca_Hành chính \\\n",
"0 False False False False \n",
"1 False False False True \n",
"2 False True False False \n",
"3 False False False False \n",
"4 False False True False \n",
"5 True False False False \n",
"6 False False False True \n",
"7 False False False True \n",
"8 True False False False \n",
"9 False False True False \n",
"\n",
" loai_ca_Part time \n",
"0 True \n",
"1 False \n",
"2 False \n",
"3 False \n",
"4 False \n",
"5 False \n",
"6 False \n",
"7 False \n",
"8 False \n",
"9 False \n",
"\n",
"[10 rows x 48 columns]"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"TARGET = \"so_luong\"\n",
"GROUP_COL = \"ma_dia_diem\"\n",
"\n",
"# chỉ bỏ 4 cột bạn yêu cầu\n",
"DROP_COLS = [\"ma_dia_diem\", \"all_task_normal\", \"all_task_dinhky\", \"is_tasks_text_missing\"]\n",
"\n",
"# ---------- helpers ----------\n",
"def parse_hour(t):\n",
" \"\"\"Convert 'HH:MM:SS' (or datetime-like) -> float hour in [0,24).\"\"\"\n",
" if pd.isna(t):\n",
" return np.nan\n",
" # pandas Timestamp / datetime\n",
" if hasattr(t, \"hour\"):\n",
" return float(t.hour) + float(getattr(t, \"minute\", 0))/60.0\n",
" s = str(t).strip()\n",
" # handle '06:30:00'\n",
" if \":\" in s:\n",
" parts = s.split(\":\")\n",
" try:\n",
" hh = int(float(parts[0]))\n",
" mm = int(float(parts[1])) if len(parts) > 1 else 0\n",
" return hh + mm/60.0\n",
" except:\n",
" return np.nan\n",
" # handle '6.5' etc\n",
" try:\n",
" return float(s)\n",
" except:\n",
" return np.nan\n",
"\n",
"# ---------- 1) y + groups ----------\n",
"y = df[TARGET].astype(float).copy()\n",
"groups = df[GROUP_COL].astype(str).copy() # để split theo tòa ở cell sau\n",
"\n",
"# ---------- 2) time features (bat_dau/ket_thuc) ----------\n",
"hour_start = df[\"bat_dau\"].apply(parse_hour)\n",
"hour_end = df[\"ket_thuc\"].apply(parse_hour)\n",
"\n",
"# cross-day (vd 22 -> 6)\n",
"is_cross_day = ((hour_end < hour_start) & hour_start.notna() & hour_end.notna()).astype(int)\n",
"\n",
"# shift length in hours, safe modulo 24\n",
"shift_length = ((hour_end - hour_start) % 24).fillna(0)\n",
"\n",
"# flags\n",
"is_night_shift = ((hour_start >= 22) | (hour_start < 6)).fillna(False).astype(int)\n",
"is_morning_shift = ((hour_start >= 6) & (hour_start < 12)).fillna(False).astype(int)\n",
"is_afternoon_shift = ((hour_start >= 12) & (hour_start < 18)).fillna(False).astype(int)\n",
"is_evening_shift = ((hour_start >= 18) & (hour_start < 22)).fillna(False).astype(int)\n",
"\n",
"# ---------- 3) Build X from df: drop forbidden + drop raw time cols + add time features ----------\n",
"X = df.drop(columns=[c for c in DROP_COLS if c in df.columns] + [TARGET], errors=\"ignore\").copy()\n",
"\n",
"# Drop raw time columns (không one-hot time)\n",
"for c in [\"bat_dau\", \"ket_thuc\"]:\n",
" if c in X.columns:\n",
" X = X.drop(columns=[c])\n",
"\n",
"# Add engineered time features\n",
"X[\"hour_start\"] = hour_start.fillna(0)\n",
"X[\"hour_end\"] = hour_end.fillna(0)\n",
"X[\"shift_length\"] = shift_length\n",
"X[\"is_cross_day\"] = is_cross_day\n",
"X[\"is_night_shift\"] = is_night_shift\n",
"X[\"is_morning_shift\"] = is_morning_shift\n",
"X[\"is_afternoon_shift\"] = is_afternoon_shift\n",
"X[\"is_evening_shift\"] = is_evening_shift\n",
"\n",
"# ---------- 4) Fill NA numeric + one-hot ONLY loai_ca ----------\n",
"# numeric fill\n",
"num_cols = X.select_dtypes(include=[np.number]).columns\n",
"X[num_cols] = X[num_cols].fillna(0)\n",
"\n",
"# One-hot loai_ca (nếu có)\n",
"if \"loai_ca\" in X.columns:\n",
" X[\"loai_ca\"] = X[\"loai_ca\"].fillna(\"UNKNOWN\").astype(str)\n",
" X = pd.get_dummies(X, columns=[\"loai_ca\"], drop_first=True)\n",
"\n",
"print(\"✅ X shape:\", X.shape, \"| y shape:\", y.shape, \"| #buildings:\", groups.nunique())\n",
"print(\"Columns sample:\", list(X.columns[:15]))\n",
"print(\"Any NaN left?:\", X.isna().any().any())\n",
"\n",
"X.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "0ad8de9d",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Final X shape: (394, 45) | y shape: (394,) | #buildings: 192\n",
"Any NaN left in X?: False\n",
"Time one-hot columns (should be 0): 0\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>tong_gio_lam</th>\n",
" <th>so_ca_cua_toa</th>\n",
" <th>num_tasks</th>\n",
" <th>num_cleaning_tasks</th>\n",
" <th>num_trash_collection_tasks</th>\n",
" <th>num_monitoring_tasks</th>\n",
" <th>num_deep_cleaning_tasks</th>\n",
" <th>num_support_tasks</th>\n",
" <th>num_other_tasks</th>\n",
" <th>num_wc_tasks</th>\n",
" <th>...</th>\n",
" <th>hour_end</th>\n",
" <th>shift_length</th>\n",
" <th>is_cross_day</th>\n",
" <th>is_night_shift</th>\n",
" <th>loai_ca_Ca chiều</th>\n",
" <th>loai_ca_Ca gãy</th>\n",
" <th>loai_ca_Ca sáng</th>\n",
" <th>loai_ca_Ca đêm</th>\n",
" <th>loai_ca_Hành chính</th>\n",
" <th>loai_ca_Part time</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>4.0</td>\n",
" <td>1</td>\n",
" <td>7</td>\n",
" <td>7</td>\n",
" <td>1</td>\n",
" <td>2</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>4</td>\n",
" <td>...</td>\n",
" <td>10.5</td>\n",
" <td>4.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>7.5</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>16.0</td>\n",
" <td>9.5</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>8.0</td>\n",
" <td>6</td>\n",
" <td>441</td>\n",
" <td>258</td>\n",
" <td>145</td>\n",
" <td>134</td>\n",
" <td>75</td>\n",
" <td>57</td>\n",
" <td>45</td>\n",
" <td>89</td>\n",
" <td>...</td>\n",
" <td>14.0</td>\n",
" <td>8.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>True</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" <td>False</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3 rows × 45 columns</p>\n",
"</div>"
],
"text/plain": [
" tong_gio_lam so_ca_cua_toa num_tasks num_cleaning_tasks \\\n",
"0 4.0 1 7 7 \n",
"1 7.5 6 441 258 \n",
"2 8.0 6 441 258 \n",
"\n",
" num_trash_collection_tasks num_monitoring_tasks num_deep_cleaning_tasks \\\n",
"0 1 2 1 \n",
"1 145 134 75 \n",
"2 145 134 75 \n",
"\n",
" num_support_tasks num_other_tasks num_wc_tasks ... hour_end \\\n",
"0 0 0 4 ... 10.5 \n",
"1 57 45 89 ... 16.0 \n",
"2 57 45 89 ... 14.0 \n",
"\n",
" shift_length is_cross_day is_night_shift loai_ca_Ca chiều \\\n",
"0 4.0 0 0 False \n",
"1 9.5 0 0 False \n",
"2 8.0 0 0 False \n",
"\n",
" loai_ca_Ca gãy loai_ca_Ca sáng loai_ca_Ca đêm loai_ca_Hành chính \\\n",
"0 False False False False \n",
"1 False False False True \n",
"2 False True False False \n",
"\n",
" loai_ca_Part time \n",
"0 True \n",
"1 False \n",
"2 False \n",
"\n",
"[3 rows x 45 columns]"
]
},
"execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"\n",
"TARGET = \"so_luong\"\n",
"GROUP_COL = \"ma_dia_diem\"\n",
"DROP_COLS = [\"ma_dia_diem\", \"all_task_normal\", \"all_task_dinhky\", \"is_tasks_text_missing\"]\n",
"\n",
"def parse_hour(t):\n",
" \"\"\"Convert 'HH:MM:SS' or datetime-like -> float hour.\"\"\"\n",
" if pd.isna(t):\n",
" return np.nan\n",
" if hasattr(t, \"hour\"):\n",
" return float(t.hour) + float(getattr(t, \"minute\", 0))/60.0\n",
" s = str(t).strip()\n",
" if \":\" in s:\n",
" parts = s.split(\":\")\n",
" try:\n",
" hh = int(float(parts[0]))\n",
" mm = int(float(parts[1])) if len(parts) > 1 else 0\n",
" return hh + mm/60.0\n",
" except:\n",
" return np.nan\n",
" try:\n",
" return float(s)\n",
" except:\n",
" return np.nan\n",
"\n",
"# 1) y + groups (groups dùng ở cell 3 để split theo tòa)\n",
"y = df[TARGET].astype(float).copy()\n",
"groups = df[GROUP_COL].astype(str).copy()\n",
"\n",
"# 2) Time features từ bat_dau/ket_thuc (KHÔNG one-hot)\n",
"hour_start = df[\"bat_dau\"].apply(parse_hour)\n",
"hour_end = df[\"ket_thuc\"].apply(parse_hour)\n",
"\n",
"is_cross_day = ((hour_end < hour_start) & hour_start.notna() & hour_end.notna()).astype(int)\n",
"shift_length = ((hour_end - hour_start) % 24).fillna(0)\n",
"is_night_shift = ((hour_start >= 22) | (hour_start < 6)).fillna(False).astype(int)\n",
"\n",
"# 3) Build X: drop 4 cols + drop target + drop raw time cols\n",
"X = df.drop(columns=[c for c in DROP_COLS if c in df.columns] + [TARGET], errors=\"ignore\").copy()\n",
"X = X.drop(columns=[c for c in [\"bat_dau\", \"ket_thuc\"] if c in X.columns], errors=\"ignore\")\n",
"\n",
"# Add engineered time cols\n",
"X[\"hour_start\"] = hour_start.fillna(0)\n",
"X[\"hour_end\"] = hour_end.fillna(0)\n",
"X[\"shift_length\"] = shift_length\n",
"X[\"is_cross_day\"] = is_cross_day\n",
"X[\"is_night_shift\"] = is_night_shift\n",
"\n",
"# 4) Fill NA numeric\n",
"num_cols = X.select_dtypes(include=[np.number]).columns\n",
"X[num_cols] = X[num_cols].fillna(0)\n",
"\n",
"# 5) One-hot ONLY loai_ca (nếu có)\n",
"if \"loai_ca\" in X.columns:\n",
" X[\"loai_ca\"] = X[\"loai_ca\"].fillna(\"UNKNOWN\").astype(str)\n",
" X = pd.get_dummies(X, columns=[\"loai_ca\"], drop_first=True)\n",
"\n",
"print(\"✅ Final X shape:\", X.shape, \"| y shape:\", y.shape, \"| #buildings:\", groups.nunique())\n",
"print(\"Any NaN left in X?:\", X.isna().any().any())\n",
"\n",
"# sanity: confirm no ket_thuc_* or bat_dau_* one-hot columns\n",
"bad_cols = [c for c in X.columns if str(c).startswith(\"bat_dau_\") or str(c).startswith(\"ket_thuc_\")]\n",
"print(\"Time one-hot columns (should be 0):\", len(bad_cols))\n",
"\n",
"X.head(3)\n"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "2df3b609",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Shapes:\n",
"Train: (283, 45) (283,) | buildings: 134\n",
"Val: (58, 45) (58,) | buildings: 29\n",
"Test: (53, 45) (53,) | buildings: 29\n",
"\n",
"Leakage check (should all be 0):\n",
"Train ∩ Val : 0\n",
"Train ∩ Test: 0\n",
"Val ∩ Test: 0\n"
]
}
],
"source": [
"from sklearn.model_selection import GroupShuffleSplit\n",
"\n",
"# ----- 1) Split: (train+val) vs test = 85% / 15% -----\n",
"gss1 = GroupShuffleSplit(n_splits=1, test_size=0.15, random_state=42)\n",
"trainval_idx, test_idx = next(gss1.split(X, y, groups=groups))\n",
"\n",
"X_trainval, X_test = X.iloc[trainval_idx].reset_index(drop=True), X.iloc[test_idx].reset_index(drop=True)\n",
"y_trainval, y_test = y.iloc[trainval_idx].reset_index(drop=True), y.iloc[test_idx].reset_index(drop=True)\n",
"groups_trainval = groups.iloc[trainval_idx].reset_index(drop=True)\n",
"groups_test = groups.iloc[test_idx].reset_index(drop=True)\n",
"\n",
"# ----- 2) Split: train vs val inside trainval = 82.35% / 17.65% -> overall 70% / 15% -----\n",
"gss2 = GroupShuffleSplit(n_splits=1, test_size=0.1765, random_state=42)\n",
"train_idx, val_idx = next(gss2.split(X_trainval, y_trainval, groups=groups_trainval))\n",
"\n",
"X_train, X_val = X_trainval.iloc[train_idx].reset_index(drop=True), X_trainval.iloc[val_idx].reset_index(drop=True)\n",
"y_train, y_val = y_trainval.iloc[train_idx].reset_index(drop=True), y_trainval.iloc[val_idx].reset_index(drop=True)\n",
"groups_train = groups_trainval.iloc[train_idx].reset_index(drop=True)\n",
"groups_val = groups_trainval.iloc[val_idx].reset_index(drop=True)\n",
"\n",
"# ----- 3) Report -----\n",
"print(\"Shapes:\")\n",
"print(\"Train:\", X_train.shape, y_train.shape, \"| buildings:\", groups_train.nunique())\n",
"print(\"Val: \", X_val.shape, y_val.shape, \"| buildings:\", groups_val.nunique())\n",
"print(\"Test: \", X_test.shape, y_test.shape, \"| buildings:\", groups_test.nunique())\n",
"\n",
"# Leakage check: ensure no building appears in multiple splits\n",
"train_b = set(groups_train.unique())\n",
"val_b = set(groups_val.unique())\n",
"test_b = set(groups_test.unique())\n",
"print(\"\\nLeakage check (should all be 0):\")\n",
"print(\"Train ∩ Val :\", len(train_b & val_b))\n",
"print(\"Train ∩ Test:\", len(train_b & test_b))\n",
"print(\"Val ∩ Test:\", len(val_b & test_b))\n"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "8cc64019",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"Model: Ridge_log\n",
"Train | MAE=2.995 | RMSE=5.484 | R2=0.484\n",
"Val | MAE=1.398 | RMSE=2.015 | R2=0.037\n",
"Test | MAE=2.744 | RMSE=4.416 | R2=0.155\n",
"\n",
"Model: GBR_log\n",
"Train | MAE=1.201 | RMSE=2.466 | R2=0.896\n",
"Val | MAE=1.213 | RMSE=1.832 | R2=0.203\n",
"Test | MAE=2.979 | RMSE=4.810 | R2=-0.002\n",
"\n",
"✅ Saved: test_predictions_gbr.xlsx\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>y_true</th>\n",
" <th>y_pred</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>29.0</td>\n",
" <td>17.014321</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4.0</td>\n",
" <td>5.206821</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.0</td>\n",
" <td>2.930329</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>1.853829</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.0</td>\n",
" <td>1.561030</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>1.0</td>\n",
" <td>0.867534</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>3.0</td>\n",
" <td>1.601529</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>3.0</td>\n",
" <td>2.436945</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>13.0</td>\n",
" <td>3.495240</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>2.0</td>\n",
" <td>1.596085</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" y_true y_pred\n",
"0 29.0 17.014321\n",
"1 4.0 5.206821\n",
"2 4.0 2.930329\n",
"3 3.0 1.853829\n",
"4 2.0 1.561030\n",
"5 1.0 0.867534\n",
"6 3.0 1.601529\n",
"7 3.0 2.436945\n",
"8 13.0 3.495240\n",
"9 2.0 1.596085"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"from sklearn.linear_model import Ridge\n",
"from sklearn.ensemble import GradientBoostingRegressor\n",
"\n",
"def rmse(y, yhat):\n",
" return float(np.sqrt(mean_squared_error(y, yhat)))\n",
"\n",
"def eval_model(name, model, X_tr, y_tr, X_va, y_va, X_te, y_te, log_target=False):\n",
" # fit\n",
" model.fit(X_tr, np.log1p(y_tr) if log_target else y_tr)\n",
"\n",
" # predict helper\n",
" def pred(m, X):\n",
" p = m.predict(X)\n",
" return np.expm1(p) if log_target else p\n",
"\n",
" yhat_tr = pred(model, X_tr)\n",
" yhat_va = pred(model, X_va)\n",
" yhat_te = pred(model, X_te)\n",
"\n",
" def metrics(y, yhat):\n",
" return {\n",
" \"MAE\": float(mean_absolute_error(y, yhat)),\n",
" \"RMSE\": rmse(y, yhat),\n",
" \"R2\": float(r2_score(y, yhat)),\n",
" }\n",
"\n",
" res = {\n",
" \"model\": name,\n",
" \"Train\": metrics(y_tr, yhat_tr),\n",
" \"Val\": metrics(y_va, yhat_va),\n",
" \"Test\": metrics(y_te, yhat_te),\n",
" }\n",
" return res, yhat_te\n",
"\n",
"\n",
"results = []\n",
"\n",
"# 1) Ridge (log target)\n",
"ridge = Ridge(alpha=1.0, random_state=42)\n",
"res_ridge, ridge_test_pred = eval_model(\n",
" \"Ridge_log\", ridge,\n",
" X_train, y_train,\n",
" X_val, y_val,\n",
" X_test, y_test,\n",
" log_target=True\n",
")\n",
"results.append(res_ridge)\n",
"\n",
"# 2) Gradient Boosting (log target)\n",
"gbr = GradientBoostingRegressor(\n",
" n_estimators=300,\n",
" learning_rate=0.05,\n",
" max_depth=3,\n",
" random_state=42\n",
")\n",
"res_gbr, gbr_test_pred = eval_model(\n",
" \"GBR_log\", gbr,\n",
" X_train, y_train,\n",
" X_val, y_val,\n",
" X_test, y_test,\n",
" log_target=True\n",
")\n",
"results.append(res_gbr)\n",
"\n",
"# Print results\n",
"for r in results:\n",
" print(\"\\nModel:\", r[\"model\"])\n",
" for split in [\"Train\", \"Val\", \"Test\"]:\n",
" m = r[split]\n",
" print(f\"{split:5s} | MAE={m['MAE']:.3f} | RMSE={m['RMSE']:.3f} | R2={m['R2']:.3f}\")\n",
"\n",
"# Save TEST predictions (GBR)\n",
"out = pd.DataFrame({\n",
" \"y_true\": y_test.values,\n",
" \"y_pred\": np.maximum(0, gbr_test_pred) # clamp negative\n",
"})\n",
"out.to_excel(\"test_predictions_gbr.xlsx\", index=False)\n",
"print(\"\\n✅ Saved: test_predictions_gbr.xlsx\")\n",
"out.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": 20,
"id": "e238b641",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"=== ML only (GBR_reg, log target) ===\n",
"Train | MAE=2.145 | RMSE=4.539 | R2=0.647\n",
"Val | MAE=1.074 | RMSE=1.722 | R2=0.296\n",
"Test | MAE=2.669 | RMSE=4.565 | R2=0.097\n"
]
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.ensemble import GradientBoostingRegressor\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"def rmse(y, yhat):\n",
" return float(np.sqrt(mean_squared_error(y, yhat)))\n",
"\n",
"def report(name, y_true, y_pred):\n",
" print(f\"{name:5s} | MAE={mean_absolute_error(y_true, y_pred):.3f} \"\n",
" f\"| RMSE={rmse(y_true, y_pred):.3f} | R2={r2_score(y_true, y_pred):.3f}\")\n",
"\n",
"# GBR \"hiền\" hơn để giảm overfit\n",
"gbr_reg = GradientBoostingRegressor(\n",
" n_estimators=600,\n",
" learning_rate=0.03,\n",
" max_depth=2,\n",
" min_samples_leaf=10,\n",
" min_samples_split=20,\n",
" random_state=42\n",
")\n",
"\n",
"# Train với log1p(target)\n",
"gbr_reg.fit(X_train, np.log1p(y_train))\n",
"\n",
"def predict_original_scale(model, X):\n",
" return np.maximum(0, np.expm1(model.predict(X)))\n",
"\n",
"pred_train = predict_original_scale(gbr_reg, X_train)\n",
"pred_val = predict_original_scale(gbr_reg, X_val)\n",
"pred_test = predict_original_scale(gbr_reg, X_test)\n",
"\n",
"print(\"=== ML only (GBR_reg, log target) ===\")\n",
"report(\"Train\", y_train, pred_train)\n",
"report(\"Val\", y_val, pred_val)\n",
"report(\"Test\", y_test, pred_test)\n"
]
},
{
"cell_type": "code",
"execution_count": 21,
"id": "635bf672",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Best K on VAL: {'K': 150, 'mae_val': 1.0616528195190562}\n",
"\n",
"=== After Rule: y_pred_final = max(ML_pred, ceil(num_tasks/K)) ===\n",
"Rule used: min_staff = ceil(num_tasks / 150)\n",
"Train | MAE=2.259 | RMSE=4.601 | R2=0.637\n",
"Val | MAE=1.062 | RMSE=1.721 | R2=0.297\n",
"Test | MAE=2.602 | RMSE=4.527 | R2=0.112\n",
"\n",
"TEST big cases (y_true >= 10): 6\n",
"ML | MAE=8.512 | RMSE=9.050 | R2=-0.959\n",
"Rule | MAE=8.320 | RMSE=8.909 | R2=-0.899\n",
"\n",
"✅ Saved: test_predictions_ml_plus_rule.xlsx\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>y_true</th>\n",
" <th>y_pred_ml</th>\n",
" <th>y_pred_final</th>\n",
" <th>num_tasks</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>29.0</td>\n",
" <td>15.005548</td>\n",
" <td>15.005548</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4.0</td>\n",
" <td>4.369563</td>\n",
" <td>4.369563</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.0</td>\n",
" <td>2.902404</td>\n",
" <td>4.000000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>2.591762</td>\n",
" <td>4.000000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.0</td>\n",
" <td>2.089570</td>\n",
" <td>2.089570</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>1.0</td>\n",
" <td>0.946909</td>\n",
" <td>1.000000</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>3.0</td>\n",
" <td>1.167053</td>\n",
" <td>2.000000</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>3.0</td>\n",
" <td>2.154357</td>\n",
" <td>2.154357</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>13.0</td>\n",
" <td>3.034284</td>\n",
" <td>3.034284</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>2.0</td>\n",
" <td>1.674387</td>\n",
" <td>1.674387</td>\n",
" <td>124</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" y_true y_pred_ml y_pred_final num_tasks\n",
"0 29.0 15.005548 15.005548 593\n",
"1 4.0 4.369563 4.369563 593\n",
"2 4.0 2.902404 4.000000 593\n",
"3 3.0 2.591762 4.000000 593\n",
"4 2.0 2.089570 2.089570 33\n",
"5 1.0 0.946909 1.000000 33\n",
"6 3.0 1.167053 2.000000 182\n",
"7 3.0 2.154357 2.154357 182\n",
"8 13.0 3.034284 3.034284 182\n",
"9 2.0 1.674387 1.674387 124"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"def rmse(y, yhat):\n",
" return float(np.sqrt(mean_squared_error(y, yhat)))\n",
"\n",
"def report(name, y_true, y_pred):\n",
" print(f\"{name:5s} | MAE={mean_absolute_error(y_true, y_pred):.3f} \"\n",
" f\"| RMSE={rmse(y_true, y_pred):.3f} | R2={r2_score(y_true, y_pred):.3f}\")\n",
"\n",
"# num_tasks phải tồn tại trong X_train/X_val/X_test\n",
"assert \"num_tasks\" in X_train.columns, \"❌ Missing num_tasks in X_train\"\n",
"\n",
"nt_train = X_train[\"num_tasks\"].values\n",
"nt_val = X_val[\"num_tasks\"].values\n",
"nt_test = X_test[\"num_tasks\"].values\n",
"\n",
"# ---- tune K on VAL (KHÔNG đụng test khi tune) ----\n",
"Ks = [30, 40, 50, 60, 70, 80, 100, 120, 150]\n",
"best = None\n",
"\n",
"for K in Ks:\n",
" min_val = np.ceil(nt_val / K)\n",
" pred_val_rule = np.maximum(pred_val, min_val)\n",
" mae = mean_absolute_error(y_val, pred_val_rule)\n",
" if (best is None) or (mae < best[\"mae_val\"]):\n",
" best = {\"K\": K, \"mae_val\": mae}\n",
"\n",
"print(\"Best K on VAL:\", best)\n",
"\n",
"K_best = best[\"K\"]\n",
"\n",
"def apply_rule(pred, num_tasks, K):\n",
" min_staff = np.ceil(num_tasks / K)\n",
" return np.maximum(pred, min_staff)\n",
"\n",
"pred_train_rule = apply_rule(pred_train, nt_train, K_best)\n",
"pred_val_rule = apply_rule(pred_val, nt_val, K_best)\n",
"pred_test_rule = apply_rule(pred_test, nt_test, K_best)\n",
"\n",
"print(\"\\n=== After Rule: y_pred_final = max(ML_pred, ceil(num_tasks/K)) ===\")\n",
"print(\"Rule used: min_staff = ceil(num_tasks / %d)\" % K_best)\n",
"\n",
"report(\"Train\", y_train, pred_train_rule)\n",
"report(\"Val\", y_val, pred_val_rule)\n",
"report(\"Test\", y_test, pred_test_rule)\n",
"\n",
"# ---- big cases analysis on TEST ----\n",
"mask_big = (y_test.values >= 10)\n",
"print(\"\\nTEST big cases (y_true >= 10):\", int(mask_big.sum()))\n",
"if mask_big.sum() > 0:\n",
" report(\"ML\", y_test.values[mask_big], pred_test[mask_big])\n",
" report(\"Rule\", y_test.values[mask_big], pred_test_rule[mask_big])\n",
"\n",
"# save test predictions (after rule)\n",
"out_rule = pd.DataFrame({\n",
" \"y_true\": y_test.values,\n",
" \"y_pred_ml\": pred_test,\n",
" \"y_pred_final\": pred_test_rule,\n",
" \"num_tasks\": nt_test\n",
"})\n",
"out_rule.to_excel(\"test_predictions_ml_plus_rule.xlsx\", index=False)\n",
"print(\"\\n✅ Saved: test_predictions_ml_plus_rule.xlsx\")\n",
"out_rule.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": 22,
"id": "70493591",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Best (a,b) on VAL: {'a': 0, 'b': 0.1, 'mae_val': 1.0737151241156944}\n",
"\n",
"=== After Smooth Rule: y_pred_final = max(ML_pred, ceil(a + b*sqrt(num_tasks))) ===\n",
"Rule used: ceil(0 + 0.1*sqrt(num_tasks))\n",
"Train | MAE=2.185 | RMSE=4.551 | R2=0.645\n",
"Val | MAE=1.074 | RMSE=1.701 | R2=0.314\n",
"Test | MAE=2.599 | RMSE=4.556 | R2=0.101\n",
"\n",
"TEST big cases (y_true >= 10): 6\n",
"ML | MAE=8.512 | RMSE=9.050 | R2=-0.959\n",
"Smooth | MAE=8.512 | RMSE=9.050 | R2=-0.959\n",
"\n",
"✅ Saved: test_predictions_ml_plus_smooth_rule.xlsx\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>y_true</th>\n",
" <th>y_pred_ml</th>\n",
" <th>y_pred_final</th>\n",
" <th>num_tasks</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>29.0</td>\n",
" <td>15.005548</td>\n",
" <td>15.005548</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4.0</td>\n",
" <td>4.369563</td>\n",
" <td>4.369563</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.0</td>\n",
" <td>2.902404</td>\n",
" <td>3.000000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>2.591762</td>\n",
" <td>3.000000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.0</td>\n",
" <td>2.089570</td>\n",
" <td>2.089570</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>1.0</td>\n",
" <td>0.946909</td>\n",
" <td>1.000000</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>3.0</td>\n",
" <td>1.167053</td>\n",
" <td>2.000000</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>3.0</td>\n",
" <td>2.154357</td>\n",
" <td>2.154357</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>13.0</td>\n",
" <td>3.034284</td>\n",
" <td>3.034284</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>2.0</td>\n",
" <td>1.674387</td>\n",
" <td>2.000000</td>\n",
" <td>124</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" y_true y_pred_ml y_pred_final num_tasks\n",
"0 29.0 15.005548 15.005548 593\n",
"1 4.0 4.369563 4.369563 593\n",
"2 4.0 2.902404 3.000000 593\n",
"3 3.0 2.591762 3.000000 593\n",
"4 2.0 2.089570 2.089570 33\n",
"5 1.0 0.946909 1.000000 33\n",
"6 3.0 1.167053 2.000000 182\n",
"7 3.0 2.154357 2.154357 182\n",
"8 13.0 3.034284 3.034284 182\n",
"9 2.0 1.674387 2.000000 124"
]
},
"execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"def rmse(y, yhat):\n",
" return float(np.sqrt(mean_squared_error(y, yhat)))\n",
"\n",
"def report(name, y_true, y_pred):\n",
" print(f\"{name:5s} | MAE={mean_absolute_error(y_true, y_pred):.3f} \"\n",
" f\"| RMSE={rmse(y_true, y_pred):.3f} | R2={r2_score(y_true, y_pred):.3f}\")\n",
"\n",
"nt_train = X_train[\"num_tasks\"].values\n",
"nt_val = X_val[\"num_tasks\"].values\n",
"nt_test = X_test[\"num_tasks\"].values\n",
"\n",
"# search grid (nhỏ thôi để bạn dễ đọc)\n",
"a_list = [0, 1, 2]\n",
"b_list = [0.10, 0.15, 0.20, 0.25, 0.30]\n",
"\n",
"best = None\n",
"for a in a_list:\n",
" for b in b_list:\n",
" min_val = np.ceil(a + b * np.sqrt(nt_val))\n",
" pred_val_rule = np.maximum(pred_val, min_val)\n",
" mae = mean_absolute_error(y_val, pred_val_rule)\n",
" if (best is None) or (mae < best[\"mae_val\"]):\n",
" best = {\"a\": a, \"b\": b, \"mae_val\": mae}\n",
"\n",
"print(\"Best (a,b) on VAL:\", best)\n",
"\n",
"a_best, b_best = best[\"a\"], best[\"b\"]\n",
"\n",
"def apply_smooth_rule(pred, num_tasks, a, b):\n",
" min_staff = np.ceil(a + b * np.sqrt(num_tasks))\n",
" return np.maximum(pred, min_staff)\n",
"\n",
"pred_train_rule2 = apply_smooth_rule(pred_train, nt_train, a_best, b_best)\n",
"pred_val_rule2 = apply_smooth_rule(pred_val, nt_val, a_best, b_best)\n",
"pred_test_rule2 = apply_smooth_rule(pred_test, nt_test, a_best, b_best)\n",
"\n",
"print(\"\\n=== After Smooth Rule: y_pred_final = max(ML_pred, ceil(a + b*sqrt(num_tasks))) ===\")\n",
"print(f\"Rule used: ceil({a_best} + {b_best}*sqrt(num_tasks))\")\n",
"\n",
"report(\"Train\", y_train, pred_train_rule2)\n",
"report(\"Val\", y_val, pred_val_rule2)\n",
"report(\"Test\", y_test, pred_test_rule2)\n",
"\n",
"mask_big = (y_test.values >= 10)\n",
"print(\"\\nTEST big cases (y_true >= 10):\", int(mask_big.sum()))\n",
"if mask_big.sum() > 0:\n",
" report(\"ML\", y_test.values[mask_big], pred_test[mask_big])\n",
" report(\"Smooth\",y_test.values[mask_big], pred_test_rule2[mask_big])\n",
"\n",
"# save\n",
"out_rule2 = pd.DataFrame({\n",
" \"y_true\": y_test.values,\n",
" \"y_pred_ml\": pred_test,\n",
" \"y_pred_final\": pred_test_rule2,\n",
" \"num_tasks\": nt_test\n",
"})\n",
"out_rule2.to_excel(\"test_predictions_ml_plus_smooth_rule.xlsx\", index=False)\n",
"print(\"\\n✅ Saved: test_predictions_ml_plus_smooth_rule.xlsx\")\n",
"out_rule2.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e71605c4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Big-rate (positive class=1):\n",
"Train: 0.14840989399293286 | count: 42 / 283\n",
"Val: 0.017241379310344827 | count: 1 / 58\n",
"Test: 0.11320754716981132 | count: 6 / 53\n",
"\n",
"================================================================================\n",
"MODEL: LogReg_balanced\n",
"\n",
"[VAL] classification_report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.981 0.930 0.955 57\n",
" 1 0.000 0.000 0.000 1\n",
"\n",
" accuracy 0.914 58\n",
" macro avg 0.491 0.465 0.477 58\n",
"weighted avg 0.965 0.914 0.938 58\n",
"\n",
"VAL confusion_matrix:\n",
" [[53 4]\n",
" [ 1 0]]\n",
"\n",
"[TEST] classification_report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.881 0.787 0.831 47\n",
" 1 0.091 0.167 0.118 6\n",
"\n",
" accuracy 0.717 53\n",
" macro avg 0.486 0.477 0.475 53\n",
"weighted avg 0.792 0.717 0.751 53\n",
"\n",
"TEST confusion_matrix:\n",
" [[37 10]\n",
" [ 5 1]]\n",
"\n",
"================================================================================\n",
"MODEL: GBC_depth2\n",
"\n",
"[VAL] classification_report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.983 1.000 0.991 57\n",
" 1 0.000 0.000 0.000 1\n",
"\n",
" accuracy 0.983 58\n",
" macro avg 0.491 0.500 0.496 58\n",
"weighted avg 0.966 0.983 0.974 58\n",
"\n",
"VAL confusion_matrix:\n",
" [[57 0]\n",
" [ 1 0]]\n",
"\n",
"[TEST] classification_report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.907 0.830 0.867 47\n",
" 1 0.200 0.333 0.250 6\n",
"\n",
" accuracy 0.774 53\n",
" macro avg 0.553 0.582 0.558 53\n",
"weighted avg 0.827 0.774 0.797 53\n",
"\n",
"TEST confusion_matrix:\n",
" [[39 8]\n",
" [ 4 2]]\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
]
}
],
"source": [
"# Liệt kê các biến model đang tồn tại\n",
"[name for name in globals().keys() if \"gbr\" in name.lower()]\n"
]
},
{
"cell_type": "code",
"execution_count": 25,
"id": "ce971deb",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"['gbr', 'res_gbr', 'gbr_test_pred', 'gbr_reg']"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Liệt kê các biến model đang tồn tại\n",
"[name for name in globals().keys() if \"gbr\" in name.lower()]\n"
]
},
{
"cell_type": "code",
"execution_count": 26,
"id": "9ad22a15",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== TEST RESULTS ===\n",
"ML only | MAE=2.669 | RMSE=4.565 | R2=0.097\n",
"ML + Business Rules | MAE=2.663 | RMSE=4.510 | R2=0.119\n",
"✅ Saved: test_predictions_ml_plus_business_rules.xlsx\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>y_true</th>\n",
" <th>y_pred_ml</th>\n",
" <th>min_staff_rule</th>\n",
" <th>y_pred_final</th>\n",
" <th>num_tasks</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>29.0</td>\n",
" <td>15.006</td>\n",
" <td>4.0</td>\n",
" <td>15.006</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4.0</td>\n",
" <td>4.370</td>\n",
" <td>4.0</td>\n",
" <td>4.370</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.0</td>\n",
" <td>2.902</td>\n",
" <td>4.0</td>\n",
" <td>4.000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>2.592</td>\n",
" <td>6.0</td>\n",
" <td>6.000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.0</td>\n",
" <td>2.090</td>\n",
" <td>1.0</td>\n",
" <td>2.090</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>1.0</td>\n",
" <td>0.947</td>\n",
" <td>1.0</td>\n",
" <td>1.000</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>3.0</td>\n",
" <td>1.167</td>\n",
" <td>3.0</td>\n",
" <td>3.000</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>3.0</td>\n",
" <td>2.154</td>\n",
" <td>2.0</td>\n",
" <td>2.154</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>13.0</td>\n",
" <td>3.034</td>\n",
" <td>2.0</td>\n",
" <td>3.034</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>2.0</td>\n",
" <td>1.674</td>\n",
" <td>3.0</td>\n",
" <td>3.000</td>\n",
" <td>124</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" y_true y_pred_ml min_staff_rule y_pred_final num_tasks\n",
"0 29.0 15.006 4.0 15.006 593\n",
"1 4.0 4.370 4.0 4.370 593\n",
"2 4.0 2.902 4.0 4.000 593\n",
"3 3.0 2.592 6.0 6.000 593\n",
"4 2.0 2.090 1.0 2.090 33\n",
"5 1.0 0.947 1.0 1.000 33\n",
"6 3.0 1.167 3.0 3.000 182\n",
"7 3.0 2.154 2.0 2.154 182\n",
"8 13.0 3.034 2.0 3.034 182\n",
"9 2.0 1.674 3.0 3.000 124"
]
},
"execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"def eval_reg(y_true, y_pred, name):\n",
" rmse = float(np.sqrt(mean_squared_error(y_true, y_pred)))\n",
" print(f\"{name:20s} | MAE={mean_absolute_error(y_true, y_pred):.3f} | RMSE={rmse:.3f} | R2={r2_score(y_true, y_pred):.3f}\")\n",
"\n",
"# -----------------------\n",
"# 1) ML prediction (log target -> original)\n",
"# -----------------------\n",
"y_pred_ml = np.maximum(0, np.expm1(gbr_reg.predict(X_test)))\n",
"\n",
"# -----------------------\n",
"# 2) Business rules: minimum staffing\n",
"# (các cột có thể có/không có nên check)\n",
"# -----------------------\n",
"min_staff = np.ceil(X_test[\"num_tasks\"] / 150)\n",
"\n",
"if \"num_wc_tasks\" in X_test.columns:\n",
" min_staff = np.maximum(min_staff, np.ceil(X_test[\"num_wc_tasks\"] / 40))\n",
"\n",
"if \"num_outdoor_tasks\" in X_test.columns:\n",
" min_staff = np.maximum(min_staff, np.ceil(X_test[\"num_outdoor_tasks\"] / 60))\n",
"\n",
"# Nếu bạn có cột shift (đêm / qua ngày) thì cộng thêm\n",
"for col in [\"is_night_shift\", \"is_cross_day\"]:\n",
" if col in X_test.columns:\n",
" min_staff = min_staff + X_test[col].astype(int)\n",
"\n",
"# -----------------------\n",
"# 3) Final prediction\n",
"# -----------------------\n",
"y_pred_final = np.maximum(y_pred_ml, min_staff)\n",
"\n",
"# -----------------------\n",
"# 4) Evaluate\n",
"# -----------------------\n",
"print(\"\\n=== TEST RESULTS ===\")\n",
"eval_reg(y_test, y_pred_ml, \"ML only\")\n",
"eval_reg(y_test, y_pred_final,\"ML + Business Rules\")\n",
"\n",
"# -----------------------\n",
"# 5) Save file\n",
"# -----------------------\n",
"out = pd.DataFrame({\n",
" \"y_true\": y_test.values,\n",
" \"y_pred_ml\": np.round(y_pred_ml, 3),\n",
" \"min_staff_rule\": min_staff.astype(float).values,\n",
" \"y_pred_final\": np.round(y_pred_final, 3),\n",
" \"num_tasks\": X_test[\"num_tasks\"].values\n",
"})\n",
"\n",
"out.to_excel(\"test_predictions_ml_plus_business_rules.xlsx\", index=False)\n",
"print(\"✅ Saved: test_predictions_ml_plus_business_rules.xlsx\")\n",
"\n",
"out.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": 27,
"id": "79387bd4",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Best rule on VAL: {'k_tasks': 150, 'k_wc': 30, 'k_out': 40, 'mae_val': 1.0984917361586377}\n",
"\n",
"=== TEST EVAL ===\n",
"ML only | MAE=2.669 | RMSE=4.565 | R2=0.097\n",
"ML + tuned business rules | MAE=2.600 | RMSE=4.440 | R2=0.146\n",
"✅ Saved: test_predictions_ml_plus_tuned_rules.xlsx\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>y_true</th>\n",
" <th>y_pred_ml</th>\n",
" <th>min_staff_rule</th>\n",
" <th>y_pred_final</th>\n",
" <th>num_tasks</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>29.0</td>\n",
" <td>15.006</td>\n",
" <td>4.0</td>\n",
" <td>15.006</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4.0</td>\n",
" <td>4.370</td>\n",
" <td>4.0</td>\n",
" <td>4.370</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.0</td>\n",
" <td>2.902</td>\n",
" <td>4.0</td>\n",
" <td>4.000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>2.592</td>\n",
" <td>6.0</td>\n",
" <td>6.000</td>\n",
" <td>593</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.0</td>\n",
" <td>2.090</td>\n",
" <td>1.0</td>\n",
" <td>2.090</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>1.0</td>\n",
" <td>0.947</td>\n",
" <td>1.0</td>\n",
" <td>1.000</td>\n",
" <td>33</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>3.0</td>\n",
" <td>1.167</td>\n",
" <td>3.0</td>\n",
" <td>3.000</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>3.0</td>\n",
" <td>2.154</td>\n",
" <td>2.0</td>\n",
" <td>2.154</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>13.0</td>\n",
" <td>3.034</td>\n",
" <td>2.0</td>\n",
" <td>3.034</td>\n",
" <td>182</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>2.0</td>\n",
" <td>1.674</td>\n",
" <td>3.0</td>\n",
" <td>3.000</td>\n",
" <td>124</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" y_true y_pred_ml min_staff_rule y_pred_final num_tasks\n",
"0 29.0 15.006 4.0 15.006 593\n",
"1 4.0 4.370 4.0 4.370 593\n",
"2 4.0 2.902 4.0 4.000 593\n",
"3 3.0 2.592 6.0 6.000 593\n",
"4 2.0 2.090 1.0 2.090 33\n",
"5 1.0 0.947 1.0 1.000 33\n",
"6 3.0 1.167 3.0 3.000 182\n",
"7 3.0 2.154 2.0 2.154 182\n",
"8 13.0 3.034 2.0 3.034 182\n",
"9 2.0 1.674 3.0 3.000 124"
]
},
"execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"def rmse(y, yhat):\n",
" return float(np.sqrt(mean_squared_error(y, yhat)))\n",
"\n",
"def eval_reg(y_true, y_pred, name):\n",
" print(f\"{name:25s} | MAE={mean_absolute_error(y_true, y_pred):.3f} | RMSE={rmse(y_true, y_pred):.3f} | R2={r2_score(y_true, y_pred):.3f}\")\n",
"\n",
"# ML preds (đã train gbr_reg)\n",
"pred_train = np.maximum(0, np.expm1(gbr_reg.predict(X_train)))\n",
"pred_val = np.maximum(0, np.expm1(gbr_reg.predict(X_val)))\n",
"pred_test = np.maximum(0, np.expm1(gbr_reg.predict(X_test)))\n",
"\n",
"def compute_min_staff(X, k_tasks, k_wc=None, k_out=None):\n",
" ms = np.ceil(X[\"num_tasks\"] / k_tasks)\n",
"\n",
" if (k_wc is not None) and (\"num_wc_tasks\" in X.columns):\n",
" ms = np.maximum(ms, np.ceil(X[\"num_wc_tasks\"] / k_wc))\n",
"\n",
" if (k_out is not None) and (\"num_outdoor_tasks\" in X.columns):\n",
" ms = np.maximum(ms, np.ceil(X[\"num_outdoor_tasks\"] / k_out))\n",
"\n",
" for col in [\"is_night_shift\", \"is_cross_day\"]:\n",
" if col in X.columns:\n",
" ms = ms + X[col].astype(int)\n",
" return ms\n",
"\n",
"# Grid search trên VAL\n",
"k_tasks_list = [100, 120, 150, 180, 200]\n",
"k_wc_list = [25, 30, 40, 50, None] # None = bỏ rule wc\n",
"k_out_list = [40, 60, 80, None] # None = bỏ rule outdoor\n",
"\n",
"best = None\n",
"\n",
"for kt in k_tasks_list:\n",
" for kw in k_wc_list:\n",
" for ko in k_out_list:\n",
" ms_val = compute_min_staff(X_val, kt, kw, ko)\n",
" pred_val_final = np.maximum(pred_val, ms_val)\n",
" mae_val = mean_absolute_error(y_val, pred_val_final)\n",
"\n",
" if (best is None) or (mae_val < best[\"mae_val\"]):\n",
" best = {\"k_tasks\": kt, \"k_wc\": kw, \"k_out\": ko, \"mae_val\": mae_val}\n",
"\n",
"print(\"✅ Best rule on VAL:\", best)\n",
"\n",
"# Apply best rule to TEST\n",
"ms_test = compute_min_staff(X_test, best[\"k_tasks\"], best[\"k_wc\"], best[\"k_out\"])\n",
"pred_test_final = np.maximum(pred_test, ms_test)\n",
"\n",
"print(\"\\n=== TEST EVAL ===\")\n",
"eval_reg(y_test, pred_test, \"ML only\")\n",
"eval_reg(y_test, pred_test_final, \"ML + tuned business rules\")\n",
"\n",
"# Save predictions\n",
"out = pd.DataFrame({\n",
" \"y_true\": y_test.values,\n",
" \"y_pred_ml\": np.round(pred_test, 3),\n",
" \"min_staff_rule\": ms_test.astype(float).values,\n",
" \"y_pred_final\": np.round(pred_test_final, 3),\n",
" \"num_tasks\": X_test[\"num_tasks\"].values\n",
"})\n",
"out.to_excel(\"test_predictions_ml_plus_tuned_rules.xlsx\", index=False)\n",
"print(\"✅ Saved: test_predictions_ml_plus_tuned_rules.xlsx\")\n",
"out.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": 28,
"id": "6e9841d0",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Class distribution:\n",
"Train:\n",
"so_luong\n",
"0 207\n",
"1 39\n",
"2 37\n",
"Name: count, dtype: int64\n",
"Val:\n",
"so_luong\n",
"0 53\n",
"1 4\n",
"2 1\n",
"Name: count, dtype: int64\n",
"Test:\n",
"so_luong\n",
"0 43\n",
"1 5\n",
"2 5\n",
"Name: count, dtype: int64\n"
]
}
],
"source": [
"# ===== CELL 9: Create staff size class =====\n",
"\n",
"def staff_class(y):\n",
" if y <= 5:\n",
" return 0\n",
" elif y <= 10:\n",
" return 1\n",
" else:\n",
" return 2\n",
"\n",
"y_train_cls = y_train.apply(staff_class)\n",
"y_val_cls = y_val.apply(staff_class)\n",
"y_test_cls = y_test.apply(staff_class)\n",
"\n",
"print(\"Class distribution:\")\n",
"for name, y in [(\"Train\", y_train_cls), (\"Val\", y_val_cls), (\"Test\", y_test_cls)]:\n",
" print(f\"{name}:\")\n",
" print(y.value_counts().sort_index())\n"
]
},
{
"cell_type": "code",
"execution_count": 29,
"id": "daf4acc7",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\linear_model\\_logistic.py:1247: FutureWarning: 'multi_class' was deprecated in version 1.5 and will be removed in 1.7. From then on, it will always use 'multinomial'. Leave it to its default value to avoid this warning.\n",
" warnings.warn(\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\linear_model\\_logistic.py:465: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
"STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT.\n",
"\n",
"Increase the number of iterations (max_iter) or scale the data as shown in:\n",
" https://scikit-learn.org/stable/modules/preprocessing.html\n",
"Please also refer to the documentation for alternative solver options:\n",
" https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
" n_iter_i = _check_optimize_result(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"=== VAL ===\n",
"\n",
"================================================================================\n",
"MODEL: LR_balanced (VAL)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.956 0.811 0.878 53\n",
" 1 0.111 0.250 0.154 4\n",
" 2 0.000 0.000 0.000 1\n",
"\n",
" accuracy 0.759 58\n",
" macro avg 0.356 0.354 0.344 58\n",
"weighted avg 0.881 0.759 0.813 58\n",
"\n",
"Confusion matrix:\n",
" [[43 7 3]\n",
" [ 2 1 1]\n",
" [ 0 1 0]]\n",
"\n",
"================================================================================\n",
"MODEL: HGB (VAL)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.946 1.000 0.972 53\n",
" 1 0.500 0.250 0.333 4\n",
" 2 0.000 0.000 0.000 1\n",
"\n",
" accuracy 0.931 58\n",
" macro avg 0.482 0.417 0.435 58\n",
"weighted avg 0.899 0.931 0.912 58\n",
"\n",
"Confusion matrix:\n",
" [[53 0 0]\n",
" [ 3 1 0]\n",
" [ 0 1 0]]\n",
"\n",
"=== TEST ===\n",
"\n",
"================================================================================\n",
"MODEL: LR_balanced (TEST)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.914 0.744 0.821 43\n",
" 1 0.231 0.600 0.333 5\n",
" 2 0.000 0.000 0.000 5\n",
"\n",
" accuracy 0.660 53\n",
" macro avg 0.382 0.448 0.385 53\n",
"weighted avg 0.764 0.660 0.697 53\n",
"\n",
"Confusion matrix:\n",
" [[32 7 4]\n",
" [ 1 3 1]\n",
" [ 2 3 0]]\n",
"\n",
"================================================================================\n",
"MODEL: HGB (TEST)\n",
" precision recall f1-score support\n",
"\n",
" 0 0.925 0.860 0.892 43\n",
" 1 0.200 0.200 0.200 5\n",
" 2 0.125 0.200 0.154 5\n",
"\n",
" accuracy 0.736 53\n",
" macro avg 0.417 0.420 0.415 53\n",
"weighted avg 0.781 0.736 0.757 53\n",
"\n",
"Confusion matrix:\n",
" [[37 2 4]\n",
" [ 1 1 3]\n",
" [ 2 2 1]]\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
]
}
],
"source": [
"import numpy as np\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.ensemble import HistGradientBoostingClassifier\n",
"from sklearn.metrics import classification_report, confusion_matrix\n",
"\n",
"# ---- 1) Models ----\n",
"clf_lr = LogisticRegression(\n",
" max_iter=5000,\n",
" class_weight=\"balanced\",\n",
" solver=\"lbfgs\",\n",
" multi_class=\"auto\",\n",
" random_state=42\n",
")\n",
"\n",
"clf_hgb = HistGradientBoostingClassifier(\n",
" max_depth=3,\n",
" learning_rate=0.05,\n",
" max_iter=500,\n",
" random_state=42\n",
")\n",
"\n",
"# ---- 2) Fit ----\n",
"clf_lr.fit(X_train, y_train_cls)\n",
"clf_hgb.fit(X_train, y_train_cls)\n",
"\n",
"# ---- 3) Predict (proba) ----\n",
"proba_val_lr = clf_lr.predict_proba(X_val)\n",
"proba_test_lr = clf_lr.predict_proba(X_test)\n",
"\n",
"proba_val_hgb = clf_hgb.predict_proba(X_val)\n",
"proba_test_hgb = clf_hgb.predict_proba(X_test)\n",
"\n",
"pred_val_lr = np.argmax(proba_val_lr, axis=1)\n",
"pred_test_lr = np.argmax(proba_test_lr, axis=1)\n",
"\n",
"pred_val_hgb = np.argmax(proba_val_hgb, axis=1)\n",
"pred_test_hgb = np.argmax(proba_test_hgb, axis=1)\n",
"\n",
"def show_clf(name, y_true, y_pred):\n",
" print(\"\\n\" + \"=\"*80)\n",
" print(\"MODEL:\", name)\n",
" print(classification_report(y_true, y_pred, digits=3))\n",
" print(\"Confusion matrix:\\n\", confusion_matrix(y_true, y_pred))\n",
"\n",
"print(\"\\n=== VAL ===\")\n",
"show_clf(\"LR_balanced (VAL)\", y_val_cls, pred_val_lr)\n",
"show_clf(\"HGB (VAL)\", y_val_cls, pred_val_hgb)\n",
"\n",
"print(\"\\n=== TEST ===\")\n",
"show_clf(\"LR_balanced (TEST)\", y_test_cls, pred_test_lr)\n",
"show_clf(\"HGB (TEST)\", y_test_cls, pred_test_hgb)\n"
]
},
{
"cell_type": "code",
"execution_count": 30,
"id": "38c8d00e",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"✅ Best (min_hc, min_2424) on VAL: {'min_hc': 0, 'min_2424': 0, 'mae_val': 1.0984917361586377}\n",
"\n",
"=== TEST EVAL ===\n",
"ML only (GBR log) | MAE=2.669 | RMSE=4.565 | R2=0.097\n",
"ML + tuned base rules | MAE=2.600 | RMSE=4.440 | R2=0.146\n",
"ML + base + loai_ca rules | MAE=2.600 | RMSE=4.440 | R2=0.146\n",
"✅ Saved: test_predictions_ml_plus_rules_plus_ca.xlsx\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>y_true</th>\n",
" <th>y_pred_ml</th>\n",
" <th>min_staff_base</th>\n",
" <th>min_staff_ca</th>\n",
" <th>min_staff_final</th>\n",
" <th>y_pred_final</th>\n",
" <th>num_tasks</th>\n",
" <th>loai_ca</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>29.0</td>\n",
" <td>15.006</td>\n",
" <td>4.0</td>\n",
" <td>0.0</td>\n",
" <td>4.0</td>\n",
" <td>15.006</td>\n",
" <td>593</td>\n",
" <td>Hành chính</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>4.0</td>\n",
" <td>4.370</td>\n",
" <td>4.0</td>\n",
" <td>0.0</td>\n",
" <td>4.0</td>\n",
" <td>4.370</td>\n",
" <td>593</td>\n",
" <td>Ca sáng</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>4.0</td>\n",
" <td>2.902</td>\n",
" <td>4.0</td>\n",
" <td>0.0</td>\n",
" <td>4.0</td>\n",
" <td>4.000</td>\n",
" <td>593</td>\n",
" <td>Ca chiều</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3.0</td>\n",
" <td>2.592</td>\n",
" <td>6.0</td>\n",
" <td>0.0</td>\n",
" <td>6.0</td>\n",
" <td>6.000</td>\n",
" <td>593</td>\n",
" <td>Ca đêm</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>2.0</td>\n",
" <td>2.090</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>2.090</td>\n",
" <td>33</td>\n",
" <td>Hành chính</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>1.0</td>\n",
" <td>0.947</td>\n",
" <td>1.0</td>\n",
" <td>0.0</td>\n",
" <td>1.0</td>\n",
" <td>1.000</td>\n",
" <td>33</td>\n",
" <td>Part time</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>3.0</td>\n",
" <td>1.167</td>\n",
" <td>3.0</td>\n",
" <td>0.0</td>\n",
" <td>3.0</td>\n",
" <td>3.000</td>\n",
" <td>182</td>\n",
" <td>Ca sáng</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>3.0</td>\n",
" <td>2.154</td>\n",
" <td>2.0</td>\n",
" <td>0.0</td>\n",
" <td>2.0</td>\n",
" <td>2.154</td>\n",
" <td>182</td>\n",
" <td>Ca chiều</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>13.0</td>\n",
" <td>3.034</td>\n",
" <td>2.0</td>\n",
" <td>0.0</td>\n",
" <td>2.0</td>\n",
" <td>3.034</td>\n",
" <td>182</td>\n",
" <td>Hành chính</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>2.0</td>\n",
" <td>1.674</td>\n",
" <td>3.0</td>\n",
" <td>0.0</td>\n",
" <td>3.0</td>\n",
" <td>3.000</td>\n",
" <td>124</td>\n",
" <td>Ca sáng</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" y_true y_pred_ml min_staff_base min_staff_ca min_staff_final \\\n",
"0 29.0 15.006 4.0 0.0 4.0 \n",
"1 4.0 4.370 4.0 0.0 4.0 \n",
"2 4.0 2.902 4.0 0.0 4.0 \n",
"3 3.0 2.592 6.0 0.0 6.0 \n",
"4 2.0 2.090 1.0 0.0 1.0 \n",
"5 1.0 0.947 1.0 0.0 1.0 \n",
"6 3.0 1.167 3.0 0.0 3.0 \n",
"7 3.0 2.154 2.0 0.0 2.0 \n",
"8 13.0 3.034 2.0 0.0 2.0 \n",
"9 2.0 1.674 3.0 0.0 3.0 \n",
"\n",
" y_pred_final num_tasks loai_ca \n",
"0 15.006 593 Hành chính \n",
"1 4.370 593 Ca sáng \n",
"2 4.000 593 Ca chiều \n",
"3 6.000 593 Ca đêm \n",
"4 2.090 33 Hành chính \n",
"5 1.000 33 Part time \n",
"6 3.000 182 Ca sáng \n",
"7 2.154 182 Ca chiều \n",
"8 3.034 182 Hành chính \n",
"9 3.000 124 Ca sáng "
]
},
"execution_count": 30,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score\n",
"\n",
"def rmse(y, yhat):\n",
" return float(np.sqrt(mean_squared_error(y, yhat)))\n",
"\n",
"def eval_reg(y_true, y_pred, name):\n",
" print(f\"{name:28s} | MAE={mean_absolute_error(y_true, y_pred):.3f} | RMSE={rmse(y_true, y_pred):.3f} | R2={r2_score(y_true, y_pred):.3f}\")\n",
"\n",
"# ========= 1) ML predictions (GBR log-target) =========\n",
"pred_train = np.maximum(0, np.expm1(gbr_reg.predict(X_train)))\n",
"pred_val = np.maximum(0, np.expm1(gbr_reg.predict(X_val)))\n",
"pred_test = np.maximum(0, np.expm1(gbr_reg.predict(X_test)))\n",
"\n",
"# ========= 2) Base min-staff rule (tuned earlier) =========\n",
"# dùng lại best bạn đã tìm: k_tasks=150, k_wc=30, k_out=40\n",
"def compute_min_staff_base(X, k_tasks=150, k_wc=30, k_out=40):\n",
" ms = np.ceil(X[\"num_tasks\"] / k_tasks)\n",
"\n",
" if \"num_wc_tasks\" in X.columns:\n",
" ms = np.maximum(ms, np.ceil(X[\"num_wc_tasks\"] / k_wc))\n",
"\n",
" if \"num_outdoor_tasks\" in X.columns:\n",
" ms = np.maximum(ms, np.ceil(X[\"num_outdoor_tasks\"] / k_out))\n",
"\n",
" # bonus theo ca đêm / cross-day nếu có\n",
" for col in [\"is_night_shift\", \"is_cross_day\"]:\n",
" if col in X.columns:\n",
" ms = ms + X[col].astype(int)\n",
"\n",
" return ms\n",
"\n",
"# ========= 3) Tune MIN staff theo loai_ca trên VAL =========\n",
"# Ta tune 2 tham số: min_HC (Hành chính), min_2424 (24/24)\n",
"# Nếu loai_ca đang one-hot, ta sẽ suy ra label từ cột gốc nếu có\n",
"def get_loai_ca_series(X):\n",
" # ưu tiên nếu còn cột gốc 'loai_ca' (string)\n",
" if \"loai_ca\" in X.columns and X[\"loai_ca\"].dtype == \"object\":\n",
" return X[\"loai_ca\"].astype(str)\n",
" # nếu đã one-hot: tìm các cột bắt đầu bằng \"loai_ca_\"\n",
" onehot_cols = [c for c in X.columns if c.startswith(\"loai_ca_\")]\n",
" if onehot_cols:\n",
" # lấy tên category có value True/1\n",
" def decode_row(row):\n",
" for c in onehot_cols:\n",
" if row[c] == 1 or row[c] is True:\n",
" return c.replace(\"loai_ca_\", \"\")\n",
" return \"UNKNOWN\"\n",
" return X[onehot_cols].apply(decode_row, axis=1)\n",
" return pd.Series([\"UNKNOWN\"] * len(X), index=X.index)\n",
"\n",
"loai_ca_val = get_loai_ca_series(X_val)\n",
"loai_ca_test = get_loai_ca_series(X_test)\n",
"loai_ca_train = get_loai_ca_series(X_train)\n",
"\n",
"def apply_ca_rule(loai_ca_series, min_hc, min_2424):\n",
" # map tên ca -> min staff\n",
" # bạn có thể thêm biến thể viết khác nếu dữ liệu có\n",
" lc = loai_ca_series.str.lower()\n",
" min_by_ca = np.zeros(len(lc), dtype=float)\n",
"\n",
" # Hành chính\n",
" mask_hc = lc.str.contains(\"hành chính\") | lc.str.contains(\"hanh chinh\")\n",
" min_by_ca[mask_hc.values] = min_hc\n",
"\n",
" # 24/24 hoặc 24-24\n",
" mask_2424 = lc.str.contains(\"24/24\") | lc.str.contains(\"24-24\") | lc.str.contains(\"24 24\")\n",
" min_by_ca[mask_2424.values] = np.maximum(min_by_ca[mask_2424.values], min_2424)\n",
"\n",
" return min_by_ca\n",
"\n",
"best = None\n",
"min_hc_list = [0, 4, 6, 8, 10]\n",
"min_2424_list = [0, 6, 8, 10, 12]\n",
"\n",
"base_val = compute_min_staff_base(X_val, 150, 30, 40)\n",
"\n",
"for mhc in min_hc_list:\n",
" for m24 in min_2424_list:\n",
" min_ca = apply_ca_rule(loai_ca_val, mhc, m24)\n",
" min_staff = np.maximum(base_val, min_ca)\n",
" pred_final = np.maximum(pred_val, min_staff)\n",
"\n",
" mae_val = mean_absolute_error(y_val, pred_final)\n",
" if (best is None) or (mae_val < best[\"mae_val\"]):\n",
" best = {\"min_hc\": mhc, \"min_2424\": m24, \"mae_val\": mae_val}\n",
"\n",
"print(\"✅ Best (min_hc, min_2424) on VAL:\", best)\n",
"\n",
"# ========= 4) Apply best rule to TEST =========\n",
"base_test = compute_min_staff_base(X_test, 150, 30, 40)\n",
"min_ca_test = apply_ca_rule(loai_ca_test, best[\"min_hc\"], best[\"min_2424\"])\n",
"min_staff_test = np.maximum(base_test, min_ca_test)\n",
"\n",
"pred_test_final = np.maximum(pred_test, min_staff_test)\n",
"\n",
"print(\"\\n=== TEST EVAL ===\")\n",
"eval_reg(y_test, pred_test, \"ML only (GBR log)\")\n",
"eval_reg(y_test, np.maximum(pred_test, base_test), \"ML + tuned base rules\")\n",
"eval_reg(y_test, pred_test_final, \"ML + base + loai_ca rules\")\n",
"\n",
"# ========= 5) Save excel =========\n",
"out = pd.DataFrame({\n",
" \"y_true\": y_test.values,\n",
" \"y_pred_ml\": np.round(pred_test, 3),\n",
" \"min_staff_base\": base_test.astype(float),\n",
" \"min_staff_ca\": min_ca_test.astype(float),\n",
" \"min_staff_final\": min_staff_test.astype(float),\n",
" \"y_pred_final\": np.round(pred_test_final, 3),\n",
" \"num_tasks\": X_test[\"num_tasks\"].values if \"num_tasks\" in X_test.columns else np.nan,\n",
" \"loai_ca\": loai_ca_test.values\n",
"})\n",
"out.to_excel(\"test_predictions_ml_plus_rules_plus_ca.xlsx\", index=False)\n",
"print(\"✅ Saved: test_predictions_ml_plus_rules_plus_ca.xlsx\")\n",
"\n",
"out.head(10)\n"
]
},
{
"cell_type": "code",
"execution_count": 32,
"id": "6dc15922",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded: final_2.xlsx | sheet: final\n",
"Shape (raw): (401, 42)\n",
"Shape (dedup): (394, 42)\n",
"\n",
"=== Target summary (so_luong) ===\n",
"count 394.000000\n",
"mean 4.710660\n",
"std 6.848602\n",
"min 0.000000\n",
"25% 1.000000\n",
"50% 2.000000\n",
"75% 5.000000\n",
"max 64.000000\n",
"Name: so_luong, dtype: float64\n",
"\n",
"=== staff_band distribution ===\n",
"staff_band\n",
"0 216\n",
"1 87\n",
"2 48\n",
"3 27\n",
"4 16\n",
"Name: count, dtype: int64\n",
"\n",
"Sample rows:\n",
" ma_dia_diem loai_ca tong_gio_lam num_tasks so_luong staff_band\n",
"0 115-2 Part time 4.0 7 1 0\n",
"1 101-1 Hành chính 7.5 441 24 4\n",
"2 101-1 Ca sáng 8.0 441 3 1\n",
"3 101-1 Ca chiều 8.0 441 5 1\n",
"4 101-1 Ca đêm 8.0 441 1 0\n",
"5 101-1 Ca gãy 7.5 441 1 0\n",
"6 101-1 Hành chính 9.5 441 22 4\n",
"7 101-2 Hành chính 9.5 135 8 2\n",
"8 101-2 Ca gãy 7.5 135 1 0\n",
"9 101-2 Ca đêm 7.5 135 1 0\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"DATA_PATH = \"final_2.xlsx\"\n",
"SHEET_NAME = \"final\"\n",
"\n",
"df = pd.read_excel(DATA_PATH, sheet_name=SHEET_NAME)\n",
"print(\"Loaded:\", DATA_PATH, \"| sheet:\", SHEET_NAME)\n",
"print(\"Shape (raw):\", df.shape)\n",
"\n",
"# drop duplicates\n",
"df = df.drop_duplicates()\n",
"print(\"Shape (dedup):\", df.shape)\n",
"\n",
"# target\n",
"assert \"so_luong\" in df.columns, \"Missing target so_luong\"\n",
"\n",
"# ---- Define ordinal bins (bậc nhân sự) ----\n",
"# 0: 0-2 (rất nhỏ)\n",
"# 1: 3-5 (nhỏ)\n",
"# 2: 6-10 (trung bình)\n",
"# 3: 11-20 (lớn)\n",
"# 4: >20 (rất lớn)\n",
"bins = [-0.1, 2, 5, 10, 20, 10**9]\n",
"labels = [0, 1, 2, 3, 4]\n",
"\n",
"df[\"staff_band\"] = pd.cut(df[\"so_luong\"], bins=bins, labels=labels).astype(int)\n",
"\n",
"print(\"\\n=== Target summary (so_luong) ===\")\n",
"print(df[\"so_luong\"].describe())\n",
"\n",
"print(\"\\n=== staff_band distribution ===\")\n",
"print(df[\"staff_band\"].value_counts().sort_index())\n",
"\n",
"print(\"\\nSample rows:\")\n",
"print(df[[\"ma_dia_diem\",\"loai_ca\",\"tong_gio_lam\",\"num_tasks\",\"so_luong\",\"staff_band\"]].head(10))\n"
]
},
{
"cell_type": "code",
"execution_count": 33,
"id": "666be810",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"=== SPLIT SUMMARY (by ma_dia_diem) ===\n",
"Buildings: 192\n",
"Train buildings: 138 | rows: 282\n",
"Val buildings: 29 | rows: 56\n",
"Test buildings: 25 | rows: 56\n",
"\n",
"Leakage check (should be 0):\n",
"Train ∩ Val : 0\n",
"Train ∩ Test: 0\n",
"Val ∩ Test: 0\n",
"\n",
"=== staff_band distribution ===\n",
"\n",
"Train:\n",
"staff_band\n",
"0 155\n",
"1 55\n",
"2 38\n",
"3 22\n",
"4 12\n",
"Name: count, dtype: int64\n",
"\n",
"Val:\n",
"staff_band\n",
"0 28\n",
"1 20\n",
"2 5\n",
"3 3\n",
"Name: count, dtype: int64\n",
"\n",
"Test:\n",
"staff_band\n",
"0 33\n",
"1 12\n",
"2 5\n",
"3 2\n",
"4 4\n",
"Name: count, dtype: int64\n"
]
}
],
"source": [
"import numpy as np\n",
"\n",
"SEED = 42\n",
"TRAIN_RATIO = 0.72\n",
"VAL_RATIO = 0.15\n",
"TEST_RATIO = 0.13\n",
"\n",
"assert abs(TRAIN_RATIO + VAL_RATIO + TEST_RATIO - 1.0) < 1e-9\n",
"\n",
"# 1) lấy danh sách tòa nhà (group)\n",
"buildings = df[\"ma_dia_diem\"].astype(str).unique()\n",
"rng = np.random.RandomState(SEED)\n",
"rng.shuffle(buildings)\n",
"\n",
"n = len(buildings)\n",
"n_train = int(round(n * TRAIN_RATIO))\n",
"n_val = int(round(n * VAL_RATIO))\n",
"# phần còn lại là test\n",
"train_b = set(buildings[:n_train])\n",
"val_b = set(buildings[n_train:n_train+n_val])\n",
"test_b = set(buildings[n_train+n_val:])\n",
"\n",
"# 2) tạo mask theo group\n",
"train_mask = df[\"ma_dia_diem\"].astype(str).isin(train_b)\n",
"val_mask = df[\"ma_dia_diem\"].astype(str).isin(val_b)\n",
"test_mask = df[\"ma_dia_diem\"].astype(str).isin(test_b)\n",
"\n",
"df_train = df[train_mask].copy()\n",
"df_val = df[val_mask].copy()\n",
"df_test = df[test_mask].copy()\n",
"\n",
"print(\"=== SPLIT SUMMARY (by ma_dia_diem) ===\")\n",
"print(\"Buildings:\", n)\n",
"print(\"Train buildings:\", len(train_b), \"| rows:\", df_train.shape[0])\n",
"print(\"Val buildings:\", len(val_b), \"| rows:\", df_val.shape[0])\n",
"print(\"Test buildings:\", len(test_b), \"| rows:\", df_test.shape[0])\n",
"\n",
"# 3) leakage check (phải = 0)\n",
"train_set = set(df_train[\"ma_dia_diem\"].astype(str).unique())\n",
"val_set = set(df_val[\"ma_dia_diem\"].astype(str).unique())\n",
"test_set = set(df_test[\"ma_dia_diem\"].astype(str).unique())\n",
"\n",
"print(\"\\nLeakage check (should be 0):\")\n",
"print(\"Train ∩ Val :\", len(train_set & val_set))\n",
"print(\"Train ∩ Test:\", len(train_set & test_set))\n",
"print(\"Val ∩ Test:\", len(val_set & test_set))\n",
"\n",
"# 4) distribution staff_band theo từng tập\n",
"print(\"\\n=== staff_band distribution ===\")\n",
"for name, d in [(\"Train\", df_train), (\"Val\", df_val), (\"Test\", df_test)]:\n",
" vc = d[\"staff_band\"].value_counts().sort_index()\n",
" print(f\"\\n{name}:\")\n",
" print(vc)\n"
]
},
{
"cell_type": "code",
"execution_count": 34,
"id": "d898cfe6",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Categorical cols: ['loai_ca', 'bat_dau', 'ket_thuc']\n",
"\n",
"Shapes:\n",
"Train: (282, 115) (282,)\n",
"Val : (56, 115) (56,)\n",
"Test : (56, 115) (56,)\n",
"\n",
"Sample feature columns (first 25):\n",
"['tong_gio_lam', 'so_ca_cua_toa', 'num_tasks', 'num_cleaning_tasks', 'num_trash_collection_tasks', 'num_monitoring_tasks', 'num_deep_cleaning_tasks', 'num_support_tasks', 'num_other_tasks', 'num_wc_tasks', 'num_hallway_tasks', 'num_lobby_tasks', 'num_outdoor_tasks', 'num_elevator_tasks', 'cleaning_ratio', 'trash_collection_ratio', 'monitoring_ratio', 'area_diversity', 'so_tang', 'so_cua_thang_may', 'dien_tich_ngoai_canh', 'dien_tich_sanh', 'dien_tich_hanh_lang', 'dien_tich_wc', 'dien_tich_phong']\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"\n",
"# ========= 1) Helper: parse time to hour float =========\n",
"def time_to_hour(x):\n",
" if pd.isna(x):\n",
" return np.nan\n",
" # nếu là kiểu datetime/time của pandas\n",
" if hasattr(x, \"hour\"):\n",
" return float(x.hour) + float(getattr(x, \"minute\", 0))/60.0\n",
" s = str(x).strip()\n",
" # format \"HH:MM:SS\" hoặc \"HH:MM\"\n",
" if \":\" in s:\n",
" parts = s.split(\":\")\n",
" try:\n",
" h = float(parts[0])\n",
" m = float(parts[1]) if len(parts) > 1 else 0.0\n",
" return h + m/60.0\n",
" except:\n",
" return np.nan\n",
" # fallback\n",
" try:\n",
" return float(s)\n",
" except:\n",
" return np.nan\n",
"\n",
"def add_time_features(d):\n",
" d = d.copy()\n",
" d[\"hour_start\"] = d[\"bat_dau\"].apply(time_to_hour)\n",
" d[\"hour_end\"] = d[\"ket_thuc\"].apply(time_to_hour)\n",
"\n",
" # shift length (handle cross-day)\n",
" # nếu end < start -> qua ngày: +24\n",
" end_adj = d[\"hour_end\"].copy()\n",
" mask_cross = (d[\"hour_end\"].notna()) & (d[\"hour_start\"].notna()) & (d[\"hour_end\"] < d[\"hour_start\"])\n",
" end_adj[mask_cross] = end_adj[mask_cross] + 24.0\n",
"\n",
" d[\"shift_length\"] = (end_adj - d[\"hour_start\"]).clip(lower=0)\n",
" d[\"is_cross_day\"] = mask_cross.astype(int)\n",
"\n",
" # buckets theo giờ bắt đầu\n",
" hs = d[\"hour_start\"].fillna(-1)\n",
" d[\"is_morning_shift\"] = ((hs >= 6) & (hs < 12)).astype(int)\n",
" d[\"is_afternoon_shift\"] = ((hs >= 12) & (hs < 18)).astype(int)\n",
" d[\"is_evening_shift\"] = ((hs >= 18) & (hs < 24)).astype(int)\n",
" d[\"is_night_shift\"] = ((hs >= 0) & (hs < 6)).astype(int)\n",
"\n",
" return d\n",
"\n",
"# ========= 2) Apply time features =========\n",
"df_train_fe = add_time_features(df_train)\n",
"df_val_fe = add_time_features(df_val)\n",
"df_test_fe = add_time_features(df_test)\n",
"\n",
"# ========= 3) Drop columns (the ones you requested) + leakage columns =========\n",
"DROP_COLS = [\"ma_dia_diem\", \"all_task_normal\", \"all_task_dinhky\", \"is_tasks_text_missing\"]\n",
"LEAK_COLS = [\"so_luong\"] # label thật -> tuyệt đối không dùng feature\n",
"\n",
"# giữ lại staff_band làm y\n",
"y_train = df_train_fe[\"staff_band\"].astype(int)\n",
"y_val = df_val_fe[\"staff_band\"].astype(int)\n",
"y_test = df_test_fe[\"staff_band\"].astype(int)\n",
"\n",
"X_train = df_train_fe.drop(columns=[c for c in (DROP_COLS + LEAK_COLS + [\"staff_band\"]) if c in df_train_fe.columns])\n",
"X_val = df_val_fe.drop(columns=[c for c in (DROP_COLS + LEAK_COLS + [\"staff_band\"]) if c in df_val_fe.columns])\n",
"X_test = df_test_fe.drop(columns=[c for c in (DROP_COLS + LEAK_COLS + [\"staff_band\"]) if c in df_test_fe.columns])\n",
"\n",
"# ========= 4) One-hot only categorical columns =========\n",
"cat_cols = [c for c in X_train.columns if X_train[c].dtype == \"object\"]\n",
"print(\"Categorical cols:\", cat_cols)\n",
"\n",
"X_train = pd.get_dummies(X_train, columns=cat_cols, dummy_na=True)\n",
"X_val = pd.get_dummies(X_val, columns=cat_cols, dummy_na=True)\n",
"X_test = pd.get_dummies(X_test, columns=cat_cols, dummy_na=True)\n",
"\n",
"# align columns across splits\n",
"X_train, X_val = X_train.align(X_val, join=\"left\", axis=1, fill_value=0)\n",
"X_train, X_test = X_train.align(X_test, join=\"left\", axis=1, fill_value=0)\n",
"\n",
"# fill NaN numeric\n",
"X_train = X_train.fillna(0)\n",
"X_val = X_val.fillna(0)\n",
"X_test = X_test.fillna(0)\n",
"\n",
"print(\"\\nShapes:\")\n",
"print(\"Train:\", X_train.shape, y_train.shape)\n",
"print(\"Val :\", X_val.shape, y_val.shape)\n",
"print(\"Test :\", X_test.shape, y_test.shape)\n",
"\n",
"print(\"\\nSample feature columns (first 25):\")\n",
"print(list(X_train.columns[:25]))\n"
]
},
{
"cell_type": "code",
"execution_count": 35,
"id": "253b34f1",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\linear_model\\_logistic.py:465: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
"STOP: TOTAL NO. OF ITERATIONS REACHED LIMIT.\n",
"\n",
"Increase the number of iterations (max_iter) or scale the data as shown in:\n",
" https://scikit-learn.org/stable/modules/preprocessing.html\n",
"Please also refer to the documentation for alternative solver options:\n",
" https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
" n_iter_i = _check_optimize_result(\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"======================================================================\n",
"MODEL: LR_balanced\n",
"Train | Acc=0.624 | MacroF1=0.552\n",
"Val | Acc=0.429 | MacroF1=0.277\n",
"Test | Acc=0.518 | MacroF1=0.402\n",
"\n",
"[VAL] Confusion matrix:\n",
"[[18 5 0 5]\n",
" [ 6 4 0 10]\n",
" [ 2 2 0 1]\n",
" [ 0 1 0 2]]\n",
"\n",
"[VAL] Classification report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.692 0.643 0.667 28\n",
" 1 0.333 0.200 0.250 20\n",
" 2 0.000 0.000 0.000 5\n",
" 3 0.111 0.667 0.190 3\n",
"\n",
" accuracy 0.429 56\n",
" macro avg 0.284 0.377 0.277 56\n",
"weighted avg 0.471 0.429 0.433 56\n",
"\n",
"\n",
"[TEST] Confusion matrix:\n",
"[[20 8 1 2 2]\n",
" [ 3 4 2 1 2]\n",
" [ 0 1 1 3 0]\n",
" [ 0 0 0 2 0]\n",
" [ 0 1 0 1 2]]\n",
"\n",
"[TEST] Classification report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.870 0.606 0.714 33\n",
" 1 0.286 0.333 0.308 12\n",
" 2 0.250 0.200 0.222 5\n",
" 3 0.222 1.000 0.364 2\n",
" 4 0.333 0.500 0.400 4\n",
"\n",
" accuracy 0.518 56\n",
" macro avg 0.392 0.528 0.402 56\n",
"weighted avg 0.628 0.518 0.548 56\n",
"\n",
"\n",
"======================================================================\n",
"MODEL: RF_balanced\n",
"Train | Acc=0.894 | MacroF1=0.875\n",
"Val | Acc=0.607 | MacroF1=0.424\n",
"Test | Acc=0.625 | MacroF1=0.453\n",
"\n",
"[VAL] Confusion matrix:\n",
"[[19 8 1 0]\n",
" [ 6 13 1 0]\n",
" [ 0 2 2 1]\n",
" [ 0 2 1 0]]\n",
"\n",
"[VAL] Classification report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.760 0.679 0.717 28\n",
" 1 0.520 0.650 0.578 20\n",
" 2 0.400 0.400 0.400 5\n",
" 3 0.000 0.000 0.000 3\n",
"\n",
" accuracy 0.607 56\n",
" macro avg 0.420 0.432 0.424 56\n",
"weighted avg 0.601 0.607 0.601 56\n",
"\n",
"\n",
"[TEST] Confusion matrix:\n",
"[[23 2 8 0 0]\n",
" [ 3 7 2 0 0]\n",
" [ 0 0 4 1 0]\n",
" [ 0 1 0 1 0]\n",
" [ 0 0 4 0 0]]\n",
"\n",
"[TEST] Classification report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.885 0.697 0.780 33\n",
" 1 0.700 0.583 0.636 12\n",
" 2 0.222 0.800 0.348 5\n",
" 3 0.500 0.500 0.500 2\n",
" 4 0.000 0.000 0.000 4\n",
"\n",
" accuracy 0.625 56\n",
" macro avg 0.461 0.516 0.453 56\n",
"weighted avg 0.709 0.625 0.645 56\n",
"\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n",
"c:\\Users\\SLG PC\\anaconda3\\Lib\\site-packages\\sklearn\\metrics\\_classification.py:1565: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
" _warn_prf(average, modifier, f\"{metric.capitalize()} is\", len(result))\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"======================================================================\n",
"MODEL: HGB\n",
"Train | Acc=0.989 | MacroF1=0.989\n",
"Val | Acc=0.589 | MacroF1=0.405\n",
"Test | Acc=0.518 | MacroF1=0.326\n",
"\n",
"[VAL] Confusion matrix:\n",
"[[22 5 1 0]\n",
" [ 8 9 2 1]\n",
" [ 1 1 2 1]\n",
" [ 1 2 0 0]]\n",
"\n",
"[VAL] Classification report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.688 0.786 0.733 28\n",
" 1 0.529 0.450 0.486 20\n",
" 2 0.400 0.400 0.400 5\n",
" 3 0.000 0.000 0.000 3\n",
"\n",
" accuracy 0.589 56\n",
" macro avg 0.404 0.409 0.405 56\n",
"weighted avg 0.569 0.589 0.576 56\n",
"\n",
"\n",
"[TEST] Confusion matrix:\n",
"[[22 6 4 1 0]\n",
" [ 4 5 3 0 0]\n",
" [ 1 2 1 1 0]\n",
" [ 0 1 1 0 0]\n",
" [ 0 1 2 0 1]]\n",
"\n",
"[TEST] Classification report:\n",
" precision recall f1-score support\n",
"\n",
" 0 0.815 0.667 0.733 33\n",
" 1 0.333 0.417 0.370 12\n",
" 2 0.091 0.200 0.125 5\n",
" 3 0.000 0.000 0.000 2\n",
" 4 1.000 0.250 0.400 4\n",
"\n",
" accuracy 0.518 56\n",
" macro avg 0.448 0.307 0.326 56\n",
"weighted avg 0.631 0.518 0.551 56\n",
"\n",
"\n",
"======================================================================\n",
"SUMMARY (sorted by Val MacroF1):\n",
"RF_balanced | Val MacroF1=0.424 | Test MacroF1=0.453\n",
"HGB | Val MacroF1=0.405 | Test MacroF1=0.326\n",
"LR_balanced | Val MacroF1=0.277 | Test MacroF1=0.402\n"
]
}
],
"source": [
"import numpy as np\n",
"from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, f1_score\n",
"from sklearn.linear_model import LogisticRegression\n",
"from sklearn.ensemble import HistGradientBoostingClassifier, RandomForestClassifier\n",
"\n",
"def eval_cls(name, model, Xtr, ytr, Xva, yva, Xte, yte):\n",
" model.fit(Xtr, ytr)\n",
"\n",
" def _metrics(split_name, X, y):\n",
" pred = model.predict(X)\n",
" acc = accuracy_score(y, pred)\n",
" f1m = f1_score(y, pred, average=\"macro\")\n",
" return pred, acc, f1m\n",
"\n",
" pred_tr, acc_tr, f1_tr = _metrics(\"Train\", Xtr, ytr)\n",
" pred_va, acc_va, f1_va = _metrics(\"Val\", Xva, yva)\n",
" pred_te, acc_te, f1_te = _metrics(\"Test\", Xte, yte)\n",
"\n",
" print(\"\\n\" + \"=\"*70)\n",
" print(f\"MODEL: {name}\")\n",
" print(f\"Train | Acc={acc_tr:.3f} | MacroF1={f1_tr:.3f}\")\n",
" print(f\"Val | Acc={acc_va:.3f} | MacroF1={f1_va:.3f}\")\n",
" print(f\"Test | Acc={acc_te:.3f} | MacroF1={f1_te:.3f}\")\n",
"\n",
" print(\"\\n[VAL] Confusion matrix:\")\n",
" print(confusion_matrix(yva, pred_va))\n",
" print(\"\\n[VAL] Classification report:\")\n",
" print(classification_report(yva, pred_va, digits=3))\n",
"\n",
" print(\"\\n[TEST] Confusion matrix:\")\n",
" print(confusion_matrix(yte, pred_te))\n",
" print(\"\\n[TEST] Classification report:\")\n",
" print(classification_report(yte, pred_te, digits=3))\n",
"\n",
" return {\n",
" \"name\": name,\n",
" \"model\": model,\n",
" \"val_macro_f1\": f1_va,\n",
" \"test_macro_f1\": f1_te\n",
" }\n",
"\n",
"# 1) Logistic Regression (balanced) - baseline mạnh cho tabular\n",
"lr = LogisticRegression(\n",
" max_iter=5000,\n",
" class_weight=\"balanced\",\n",
" n_jobs=None\n",
")\n",
"\n",
"# 2) RandomForest (balanced_subsample)\n",
"rf = RandomForestClassifier(\n",
" n_estimators=600,\n",
" max_depth=None,\n",
" min_samples_leaf=2,\n",
" random_state=42,\n",
" class_weight=\"balanced_subsample\",\n",
" n_jobs=-1\n",
")\n",
"\n",
"# 3) HistGradientBoosting (mạnh cho tabular, chạy nhanh)\n",
"hgb = HistGradientBoostingClassifier(\n",
" learning_rate=0.06,\n",
" max_depth=6,\n",
" max_iter=600,\n",
" random_state=42\n",
")\n",
"\n",
"results = []\n",
"results.append(eval_cls(\"LR_balanced\", lr, X_train, y_train, X_val, y_val, X_test, y_test))\n",
"results.append(eval_cls(\"RF_balanced\", rf, X_train, y_train, X_val, y_val, X_test, y_test))\n",
"results.append(eval_cls(\"HGB\", hgb, X_train, y_train, X_val, y_val, X_test, y_test))\n",
"\n",
"# Summary\n",
"print(\"\\n\" + \"=\"*70)\n",
"print(\"SUMMARY (sorted by Val MacroF1):\")\n",
"for r in sorted(results, key=lambda x: x[\"val_macro_f1\"], reverse=True):\n",
" print(f\"{r['name']:12s} | Val MacroF1={r['val_macro_f1']:.3f} | Test MacroF1={r['test_macro_f1']:.3f}\")\n"
]
},
{
"cell_type": "code",
"execution_count": 37,
"id": "e1851e78",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Loaded: final_2.xlsx | sheet: final\n",
"Shape (raw): (401, 42)\n",
"Shape (after dedup): (394, 42)\n",
"\n",
"=== TARGET SUMMARY (so_luong) ===\n",
"count 394.000000\n",
"mean 4.710660\n",
"std 6.848602\n",
"min 0.000000\n",
"25% 1.000000\n",
"50% 2.000000\n",
"75% 5.000000\n",
"max 64.000000\n",
"Name: so_luong, dtype: float64\n",
"Missing target: 0\n",
"Negative target: 0\n",
"Zero target: 3\n",
"\n",
"Sample rows:\n"
]
},
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>ma_dia_diem</th>\n",
" <th>all_task_normal</th>\n",
" <th>all_task_dinhky</th>\n",
" <th>loai_ca</th>\n",
" <th>bat_dau</th>\n",
" <th>ket_thuc</th>\n",
" <th>tong_gio_lam</th>\n",
" <th>so_ca_cua_toa</th>\n",
" <th>so_luong</th>\n",
" <th>num_tasks</th>\n",
" <th>...</th>\n",
" <th>dien_tich_tham</th>\n",
" <th>doc_ham</th>\n",
" <th>vien_phan_quang</th>\n",
" <th>op_tuong</th>\n",
" <th>op_chan_tuong</th>\n",
" <th>ranh_thoat_nuoc</th>\n",
" <th>dien_tich_kinh</th>\n",
" <th>num_medical_tasks_total</th>\n",
" <th>num_indoor_room_tasks</th>\n",
" <th>is_tasks_text_missing</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>115-2</td>\n",
" <td>Làm sạch toàn bộ phòng giao dịch tầng 1 (kể cả...</td>\n",
" <td>NaN</td>\n",
" <td>Part time</td>\n",
" <td>06:30:00</td>\n",
" <td>10:30:00</td>\n",
" <td>4.0</td>\n",
" <td>1</td>\n",
" <td>1</td>\n",
" <td>7</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" <td>0.0</td>\n",
" <td>0.0</td>\n",
" <td>0</td>\n",
" <td>20.0</td>\n",
" <td>0</td>\n",
" <td>1</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>101-1</td>\n",
" <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
" <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
" <td>Hành chính</td>\n",
" <td>06:30:00</td>\n",
" <td>16:00:00</td>\n",
" <td>7.5</td>\n",
" <td>6</td>\n",
" <td>24</td>\n",
" <td>441</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>70</td>\n",
" <td>0</td>\n",
" <td>9176.0</td>\n",
" <td>89.0</td>\n",
" <td>25</td>\n",
" <td>894.0</td>\n",
" <td>112</td>\n",
" <td>39</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>101-1</td>\n",
" <td>Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,...</td>\n",
" <td>Lau bảng biển, bình cứu hỏa , cây nước hành la...</td>\n",
" <td>Ca sáng</td>\n",
" <td>06:00:00</td>\n",
" <td>14:00:00</td>\n",
" <td>8.0</td>\n",
" <td>6</td>\n",
" <td>3</td>\n",
" <td>441</td>\n",
" <td>...</td>\n",
" <td>0.0</td>\n",
" <td>70</td>\n",
" <td>0</td>\n",
" <td>9176.0</td>\n",
" <td>89.0</td>\n",
" <td>25</td>\n",
" <td>894.0</td>\n",
" <td>112</td>\n",
" <td>39</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>3 rows × 42 columns</p>\n",
"</div>"
],
"text/plain": [
" ma_dia_diem all_task_normal \\\n",
"0 115-2 Làm sạch toàn bộ phòng giao dịch tầng 1 (kể cả... \n",
"1 101-1 Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,... \n",
"2 101-1 Kiểm tra nhân sự các vị trí Thay rác, đẩy khô,... \n",
"\n",
" all_task_dinhky loai_ca bat_dau \\\n",
"0 NaN Part time 06:30:00 \n",
"1 Lau bảng biển, bình cứu hỏa , cây nước hành la... Hành chính 06:30:00 \n",
"2 Lau bảng biển, bình cứu hỏa , cây nước hành la... Ca sáng 06:00:00 \n",
"\n",
" ket_thuc tong_gio_lam so_ca_cua_toa so_luong num_tasks ... \\\n",
"0 10:30:00 4.0 1 1 7 ... \n",
"1 16:00:00 7.5 6 24 441 ... \n",
"2 14:00:00 8.0 6 3 441 ... \n",
"\n",
" dien_tich_tham doc_ham vien_phan_quang op_tuong op_chan_tuong \\\n",
"0 0.0 0 0 0.0 0.0 \n",
"1 0.0 70 0 9176.0 89.0 \n",
"2 0.0 70 0 9176.0 89.0 \n",
"\n",
" ranh_thoat_nuoc dien_tich_kinh num_medical_tasks_total \\\n",
"0 0 20.0 0 \n",
"1 25 894.0 112 \n",
"2 25 894.0 112 \n",
"\n",
" num_indoor_room_tasks is_tasks_text_missing \n",
"0 1 0 \n",
"1 39 0 \n",
"2 39 0 \n",
"\n",
"[3 rows x 42 columns]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# CELL 1 — LOAD DATA & BASIC CLEAN\n",
"\n",
"import pandas as pd\n",
"\n",
"DATA_PATH = \"final_2.xlsx\"\n",
"SHEET_NAME = \"final\"\n",
"\n",
"# 1. Load\n",
"df = pd.read_excel(DATA_PATH, sheet_name=SHEET_NAME)\n",
"print(f\"Loaded: {DATA_PATH} | sheet: {SHEET_NAME}\")\n",
"print(\"Shape (raw):\", df.shape)\n",
"\n",
"# 2. Drop duplicate full rows\n",
"df = df.drop_duplicates().reset_index(drop=True)\n",
"print(\"Shape (after dedup):\", df.shape)\n",
"\n",
"# 3. Check target\n",
"assert \"so_luong\" in df.columns, \"❌ Missing target so_luong\"\n",
"\n",
"print(\"\\n=== TARGET SUMMARY (so_luong) ===\")\n",
"print(df[\"so_luong\"].describe())\n",
"print(\"Missing target:\", df[\"so_luong\"].isna().sum())\n",
"print(\"Negative target:\", (df[\"so_luong\"] < 0).sum())\n",
"print(\"Zero target:\", (df[\"so_luong\"] == 0).sum())\n",
"\n",
"# 4. Peek data\n",
"print(\"\\nSample rows:\")\n",
"display(df.head(3))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c9268c5",
"metadata": {},
"outputs": [],
"source": [
"# CELL 2 — FEATURE SELECTION (STRICT)\n",
"\n",
"# 1. Xem toàn bộ cột\n",
"print(\"All columns:\")\n",
"for i, c in enumerate(df.columns):\n",
" print(f\"{i:2d}: {c}\")\n",
"\n",
"# 2. Xác định cột cần loại bỏ (THEO THỎA THUẬN)\n",
"DROP_COLS = [\n",
" df.columns[0], # ma_dia_diem\n",
" df.columns[1], # all_task_normal\n",
" df.columns[2], # all_task_dinhky\n",
" df.columns[-1], # is_tasks_text_missing\n",
"]\n",
"\n",
"print(\"\\nDropped columns:\")\n",
"for c in DROP_COLS:\n",
" print(\" -\", c)\n",
"\n",
"# 3. Tạo X, y\n",
"X = df.drop(columns=DROP_COLS + [\"so_luong\"])\n",
"y = df[\"so_luong\"].astype(float)\n",
"\n",
"print(\"\\nShapes:\")\n",
"print(\"X:\", X.shape)\n",
"print(\"y:\", y.shape)\n",
"\n",
"# 4. Kiểm tra kiểu dữ liệu\n",
"print(\"\\nFeature dtypes:\")\n",
"display(X.dtypes.value_counts())\n",
"\n",
"# 5. Kiểm tra missing\n",
"print(\"\\nMissing values in X:\")\n",
"display(X.isna().sum().sort_values(ascending=False).head(10))\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b975f6cf",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "1a595fe8",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "b2fb9c84",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "base",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.5"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink