{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step1 导入相关包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\gitcode\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step2 加载数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Generating train split: 7766 examples [00:00, 94892.98 examples/s]\n",
      "Filter: 100%|██████████| 7766/7766 [00:00<00:00, 146401.93 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['label', 'review'],\n",
       "    num_rows: 7765\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset = load_dataset(\"csv\", \n",
    "                       data_files='D:/gitcode/data/ChnSentiCorp_htl_all.csv',\n",
    "                       split='train')\n",
    "dataset = dataset.filter(lambda x: x['review'] is not None)\n",
    "dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step3 划分数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['label', 'review'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data = dataset.train_test_split(test_size=0.1)\n",
    "train_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step4 创建Dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Map: 100%|██████████| 6988/6988 [00:00<00:00, 10228.56 examples/s]\n",
      "Map: 100%|██████████| 777/777 [00:00<00:00, 7720.36 examples/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 6988\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
       "        num_rows: 777\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"D:/gitcode/model/rbt3\")\n",
    "\n",
    "def process_fun(examples):\n",
    "    out = tokenizer(examples['review'], max_length=128, truncation=True)\n",
    "    out[\"labels\"] = list(examples[\"label\"]) \n",
    "    return out\n",
    "\n",
    "tokenized_dataset = train_data.map(process_fun, batched=True, remove_columns=train_data['train'].column_names)\n",
    "tokenized_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader\n",
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "trainset, validset = tokenized_dataset['train'], tokenized_dataset['test']\n",
    "trainloader = DataLoader(trainset, batch_size=32, shuffle=True, collate_fn=DataCollatorWithPadding(tokenizer))\n",
    "validloader = DataLoader(validset, batch_size=64, shuffle=False, collate_fn=DataCollatorWithPadding(tokenizer))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'input_ids': tensor([[ 101, 3300,  782,  ...,  705, 2168,  102],\n",
       "        [ 101, 2791, 7313,  ...,    0,    0,    0],\n",
       "        [ 101, 1728,  711,  ...,    0,    0,    0],\n",
       "        ...,\n",
       "        [ 101, 2791, 7313,  ...,    0,    0,    0],\n",
       "        [ 101,  704, 2495,  ...,    0,    0,    0],\n",
       "        [ 101, 2556,  679,  ..., 6387, 1728,  102]]), 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0],\n",
       "        [0, 0, 0,  ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        ...,\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        [1, 1, 1,  ..., 0, 0, 0],\n",
       "        [1, 1, 1,  ..., 1, 1, 1]]), 'labels': tensor([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
       "        1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1,\n",
       "        1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1])}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(enumerate(validloader))[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step5 创建模型及优化器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at D:/gitcode/model/rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from torch.optim import Adam\n",
    "import torch\n",
    "\n",
    "\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"D:/gitcode/model/rbt3\", num_labels=2)\n",
    "if torch.cuda.is_available():\n",
    "    model = model.cuda()\n",
    "\n",
    "optimizer = Adam(model.parameters(), lr=2e-5)  # 10的-5次方=0.00001 * 2 = 0.00002"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step6 训练与验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import evaluate\n",
    "\n",
    "acc_metric = evaluate.load('./evaluate-main/metrics/accuracy/accuracy.py')\n",
    "f1_metric = evaluate.load('./evaluate-main/metrics/f1/f1.py')\n",
    "\n",
    "def eval_metric(eval_predict):\n",
    "    predictions, labels = eval_predict\n",
    "    predictions = predictions.argmax(axis=-1)\n",
    "    acc = acc_metric.compute(predictions=predictions, references=labels)\n",
    "    f1 = f1_metric.compute(predictions=predictions, references=labels)\n",
    "    acc.update(f1)\n",
    "    return acc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step7 创建TrainingArguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments\n",
    "\n",
    "train_args = TrainingArguments(output_dir=\"../checkpoints\",      # 输出文件夹\n",
    "                               per_device_eval_batch_size=128,  # 验证时的batch_size\n",
    "                               per_device_train_batch_size=64,  # 训练时的batch_size\n",
    "                               logging_steps=100,                # log打印频率\n",
    "                               eval_strategy='epoch',     # 评估策略\n",
    "                               save_strategy='epoch',           # 保存策略\n",
    "                               save_total_limit=3,              # 最大保存数量\n",
    "                               learning_rate=2e-5,              # 学习率\n",
    "                               weight_decay=0.01,               # 权重衰减率\n",
    "                               metric_for_best_model='f1',      # 设定评估指标\n",
    "                               num_train_epochs=5,\n",
    "                               load_best_model_at_end=True)     # 训练完成后加载最优模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step8 创建Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\Program_Files\\Anaconda3\\envs\\tsf_v1\\lib\\site-packages\\accelerate\\accelerator.py:447: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
      "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False)\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "from transformers import DataCollatorWithPadding, Trainer\n",
    "\n",
    "trainer = Trainer(model=model, \n",
    "                 args=train_args, \n",
    "                 train_dataset=tokenized_dataset['train'], \n",
    "                 eval_dataset=tokenized_dataset['test'], \n",
    "                 data_collator=DataCollatorWithPadding(tokenizer=tokenizer),\n",
    "                 compute_metrics=eval_metric)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step9 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 101/666 [00:19<01:44,  5.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.4417, 'learning_rate': 1.6996996996997e-05, 'epoch': 0.45}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 201/666 [00:38<01:23,  5.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3713, 'learning_rate': 1.3993993993993995e-05, 'epoch': 0.9}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \n",
      " 33%|███▎      | 222/666 [00:43<01:23,  5.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3549949824810028, 'eval_accuracy': 0.8434727503168568, 'eval_f1': 0.9013972055888223, 'eval_runtime': 1.7115, 'eval_samples_per_second': 921.977, 'eval_steps_per_second': 7.596, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 301/666 [01:00<01:08,  5.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3287, 'learning_rate': 1.0990990990990992e-05, 'epoch': 1.35}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 400/666 [01:19<00:51,  5.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3258, 'learning_rate': 7.987987987987988e-06, 'epoch': 1.8}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \n",
      " 67%|██████▋   | 444/666 [01:30<00:42,  5.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.3241136074066162, 'eval_accuracy': 0.8517110266159695, 'eval_f1': 0.9042553191489362, 'eval_runtime': 1.9455, 'eval_samples_per_second': 811.122, 'eval_steps_per_second': 6.682, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 501/666 [01:42<00:31,  5.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3089, 'learning_rate': 4.984984984984985e-06, 'epoch': 2.25}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 600/666 [02:01<00:12,  5.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.2997, 'learning_rate': 1.9819819819819822e-06, 'epoch': 2.7}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \n",
      "100%|██████████| 666/666 [02:15<00:00,  5.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.32806867361068726, 'eval_accuracy': 0.858681875792142, 'eval_f1': 0.909312728751525, 'eval_runtime': 1.5521, 'eval_samples_per_second': 1016.663, 'eval_steps_per_second': 8.376, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 666/666 [02:16<00:00,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 136.3027, 'train_samples_per_second': 312.518, 'train_steps_per_second': 4.886, 'train_loss': 0.34142845314186254, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=666, training_loss=0.34142845314186254, metrics={'train_runtime': 136.3027, 'train_samples_per_second': 312.518, 'train_steps_per_second': 4.886, 'train_loss': 0.34142845314186254, 'epoch': 3.0})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step10 模型评估"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 13/13 [00:01<00:00,  9.46it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'eval_loss': 0.32806867361068726,\n",
       " 'eval_accuracy': 0.858681875792142,\n",
       " 'eval_f1': 0.909312728751525,\n",
       " 'eval_runtime': 1.513,\n",
       " 'eval_samples_per_second': 1042.948,\n",
       " 'eval_steps_per_second': 8.592,\n",
       " 'epoch': 3.0}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.evaluate(tokenized_dataset[\"test\"])\n",
    "#模型预测\n",
    "#trainer.predict(tokenized_dataset[\"test\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# step11 模型使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': '好评！', 'score': 0.9663025736808777}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import pipeline\n",
    "\n",
    "sen = \"这本书真不错！\"\n",
    "id2_label = {0: \"差评！\", 1: \"好评！\"}\n",
    "model.eval()\n",
    "model.config.id2label = id2_label\n",
    "\n",
    "pipe = pipeline(\"text-classification\", model=model, tokenizer=tokenizer, device=0)\n",
    "pipe(sen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gitcode",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}