mirrors/loss

from pathlib import Path
from utils import plot_loss, sync_logs

dirs = [
    '/home/app.e0016372/wkspace/train/pt/lora/qwen2.5-32b-instruct-pt-v1',
    '/home/app.e0016372/wkspace/train/pt/full/qwen2.5-32b-instruct-pt-v1',
    '/home/app.e0016372/wkspace/train/pt/lora/qwen2.5-32b-instruct-pt-v2',
    '/home/app.e0016372/wkspace/train/pt/full/qwen2.5-72b-instruct-pt-v1',
]

for p in dirs: plot_loss(p)

/home/app.e0016372/wkspace/train/pt/lora/qwen2.5-32b-instruct-pt-v1

current_steps	total_steps	loss	lr	epoch	percentage	elapsed_time	remaining_time	eval_loss
1200	1224	NaN	NaN	2.942042	98.04	2025-07-01 09:41:44	2025-07-01 00:11:38	1.327934
1200	1224	NaN	NaN	2.942042	98.04	2025-07-01 09:41:44	2025-07-01 00:11:38	1.327934
1224	1224	NaN	NaN	3.000000	100.00	2025-07-01 09:54:58	2025-07-01 00:00:00	NaN

png

/home/app.e0016372/wkspace/train/pt/full/qwen2.5-32b-instruct-pt-v1

current_steps	total_steps	loss	lr	epoch	percentage	elapsed_time	remaining_time	eval_loss
800	1224	NaN	NaN	1.960784	65.36	2025-07-01 08:11:34	2025-07-01 04:20:32	1.265471
1200	1224	NaN	NaN	2.941176	98.04	2025-07-01 12:20:55	2025-07-01 00:14:49	1.285147
1224	1224	NaN	NaN	3.000000	100.00	2025-07-01 12:43:48	2025-07-01 00:00:00	NaN

png

/home/app.e0016372/wkspace/train/pt/lora/qwen2.5-32b-instruct-pt-v2

current_steps	total_steps	loss	lr	epoch	percentage	elapsed_time	remaining_time	eval_loss
2000	4080	NaN	NaN	4.902791	49.02	2025-07-01 15:51:35	16:29:39	1.305410
2800	4080	NaN	NaN	6.863539	68.63	2025-07-01 22:13:47	10:09:43	1.317664
2860	4080	1.14	0.000025	7.009813	70.10	2025-07-01 22:41:51	9:40:56	NaN

png

/home/app.e0016372/wkspace/train/pt/full/qwen2.5-72b-instruct-pt-v1

current_steps	total_steps	loss	lr	epoch	percentage	elapsed_time	remaining_time	eval_loss
800	1224	NaN	NaN	1.960784	65.36	2025-07-01 08:30:11	2025-07-01 04:30:24	1.200028
1200	1224	NaN	NaN	2.941176	98.04	2025-07-01 12:50:08	2025-07-01 00:15:24	1.245664
1224	1224	NaN	NaN	3.000000	100.00	2025-07-01 13:22:32	2025-07-01 00:00:00	NaN

png

sync_logs(dirs)