test

cls_Trainer.ipynb

@@ -0,0 +1,612 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step1 导入相关包"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "d:\\gitcode\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
+    "from datasets import load_dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step2 加载数据"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Generating train split: 7766 examples [00:00, 94892.98 examples/s]\n",
+      "Filter: 100%|██████████| 7766/7766 [00:00<00:00, 146401.93 examples/s]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "Dataset({\n",
+       "    features: ['label', 'review'],\n",
+       "    num_rows: 7765\n",
+       "})"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset = load_dataset(\"csv\", \n",
+    "                       data_files='D:/gitcode/data/ChnSentiCorp_htl_all.csv',\n",
+    "                       split='train')\n",
+    "dataset = dataset.filter(lambda x: x['review'] is not None)\n",
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step3 划分数据集"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['label', 'review'],\n",
+       "        num_rows: 6988\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['label', 'review'],\n",
+       "        num_rows: 777\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "train_data = dataset.train_test_split(test_size=0.1)\n",
+    "train_data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step4 创建Dataloader"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Map: 100%|██████████| 6988/6988 [00:00<00:00, 10228.56 examples/s]\n",
+      "Map: 100%|██████████| 777/777 [00:00<00:00, 7720.36 examples/s]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
+       "        num_rows: 6988\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
+       "        num_rows: 777\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer = AutoTokenizer.from_pretrained(\"D:/gitcode/model/rbt3\")\n",
+    "\n",
+    "def process_fun(examples):\n",
+    "    out = tokenizer(examples['review'], max_length=128, truncation=True)\n",
+    "    out[\"labels\"] = list(examples[\"label\"]) \n",
+    "    return out\n",
+    "\n",
+    "tokenized_dataset = train_data.map(process_fun, batched=True, remove_columns=train_data['train'].column_names)\n",
+    "tokenized_dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from torch.utils.data import DataLoader\n",
+    "from transformers import DataCollatorWithPadding\n",
+    "\n",
+    "trainset, validset = tokenized_dataset['train'], tokenized_dataset['test']\n",
+    "trainloader = DataLoader(trainset, batch_size=32, shuffle=True, collate_fn=DataCollatorWithPadding(tokenizer))\n",
+    "validloader = DataLoader(validset, batch_size=64, shuffle=False, collate_fn=DataCollatorWithPadding(tokenizer))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': tensor([[ 101, 3300,  782,  ...,  705, 2168,  102],\n",
+       "        [ 101, 2791, 7313,  ...,    0,    0,    0],\n",
+       "        [ 101, 1728,  711,  ...,    0,    0,    0],\n",
+       "        ...,\n",
+       "        [ 101, 2791, 7313,  ...,    0,    0,    0],\n",
+       "        [ 101,  704, 2495,  ...,    0,    0,    0],\n",
+       "        [ 101, 2556,  679,  ..., 6387, 1728,  102]]), 'token_type_ids': tensor([[0, 0, 0,  ..., 0, 0, 0],\n",
+       "        [0, 0, 0,  ..., 0, 0, 0],\n",
+       "        [0, 0, 0,  ..., 0, 0, 0],\n",
+       "        ...,\n",
+       "        [0, 0, 0,  ..., 0, 0, 0],\n",
+       "        [0, 0, 0,  ..., 0, 0, 0],\n",
+       "        [0, 0, 0,  ..., 0, 0, 0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
+       "        [1, 1, 1,  ..., 0, 0, 0],\n",
+       "        [1, 1, 1,  ..., 0, 0, 0],\n",
+       "        ...,\n",
+       "        [1, 1, 1,  ..., 0, 0, 0],\n",
+       "        [1, 1, 1,  ..., 0, 0, 0],\n",
+       "        [1, 1, 1,  ..., 1, 1, 1]]), 'labels': tensor([0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
+       "        1, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1,\n",
+       "        1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1])}"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "next(enumerate(validloader))[1]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step5 创建模型及优化器"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at D:/gitcode/model/rbt3 and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "from torch.optim import Adam\n",
+    "import torch\n",
+    "\n",
+    "\n",
+    "model = AutoModelForSequenceClassification.from_pretrained(\"D:/gitcode/model/rbt3\", num_labels=2)\n",
+    "if torch.cuda.is_available():\n",
+    "    model = model.cuda()\n",
+    "\n",
+    "optimizer = Adam(model.parameters(), lr=2e-5)  # 10的-5次方=0.00001 * 2 = 0.00002"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step6 训练与验证"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import evaluate\n",
+    "\n",
+    "acc_metric = evaluate.load('./evaluate-main/metrics/accuracy/accuracy.py')\n",
+    "f1_metric = evaluate.load('./evaluate-main/metrics/f1/f1.py')\n",
+    "\n",
+    "def eval_metric(eval_predict):\n",
+    "    predictions, labels = eval_predict\n",
+    "    predictions = predictions.argmax(axis=-1)\n",
+    "    acc = acc_metric.compute(predictions=predictions, references=labels)\n",
+    "    f1 = f1_metric.compute(predictions=predictions, references=labels)\n",
+    "    acc.update(f1)\n",
+    "    return acc"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step7 创建TrainingArguments"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import TrainingArguments\n",
+    "\n",
+    "train_args = TrainingArguments(output_dir=\"../checkpoints\",      # 输出文件夹\n",
+    "                               per_device_eval_batch_size=128,  # 验证时的batch_size\n",
+    "                               per_device_train_batch_size=64,  # 训练时的batch_size\n",
+    "                               logging_steps=100,                # log打印频率\n",
+    "                               eval_strategy='epoch',     # 评估策略\n",
+    "                               save_strategy='epoch',           # 保存策略\n",
+    "                               save_total_limit=3,              # 最大保存数量\n",
+    "                               learning_rate=2e-5,              # 学习率\n",
+    "                               weight_decay=0.01,               # 权重衰减率\n",
+    "                               metric_for_best_model='f1',      # 设定评估指标\n",
+    "                               num_train_epochs=5,\n",
+    "                               load_best_model_at_end=True)     # 训练完成后加载最优模型"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step8 创建Trainer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "d:\\Program_Files\\Anaconda3\\envs\\tsf_v1\\lib\\site-packages\\accelerate\\accelerator.py:447: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches']). Please pass an `accelerate.DataLoaderConfiguration` instead: \n",
+      "dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False)\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "from transformers import DataCollatorWithPadding, Trainer\n",
+    "\n",
+    "trainer = Trainer(model=model, \n",
+    "                 args=train_args, \n",
+    "                 train_dataset=tokenized_dataset['train'], \n",
+    "                 eval_dataset=tokenized_dataset['test'], \n",
+    "                 data_collator=DataCollatorWithPadding(tokenizer=tokenizer),\n",
+    "                 compute_metrics=eval_metric)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step9 模型训练"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 15%|█▌        | 101/666 [00:19<01:44,  5.41it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.4417, 'learning_rate': 1.6996996996997e-05, 'epoch': 0.45}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 30%|███       | 201/666 [00:38<01:23,  5.57it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.3713, 'learning_rate': 1.3993993993993995e-05, 'epoch': 0.9}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                 \n",
+      " 33%|███▎      | 222/666 [00:43<01:23,  5.33it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.3549949824810028, 'eval_accuracy': 0.8434727503168568, 'eval_f1': 0.9013972055888223, 'eval_runtime': 1.7115, 'eval_samples_per_second': 921.977, 'eval_steps_per_second': 7.596, 'epoch': 1.0}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 45%|████▌     | 301/666 [01:00<01:08,  5.34it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.3287, 'learning_rate': 1.0990990990990992e-05, 'epoch': 1.35}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 60%|██████    | 400/666 [01:19<00:51,  5.20it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.3258, 'learning_rate': 7.987987987987988e-06, 'epoch': 1.8}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                 \n",
+      " 67%|██████▋   | 444/666 [01:30<00:42,  5.25it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.3241136074066162, 'eval_accuracy': 0.8517110266159695, 'eval_f1': 0.9042553191489362, 'eval_runtime': 1.9455, 'eval_samples_per_second': 811.122, 'eval_steps_per_second': 6.682, 'epoch': 2.0}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 75%|███████▌  | 501/666 [01:42<00:31,  5.26it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.3089, 'learning_rate': 4.984984984984985e-06, 'epoch': 2.25}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      " 90%|█████████ | 600/666 [02:01<00:12,  5.15it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'loss': 0.2997, 'learning_rate': 1.9819819819819822e-06, 'epoch': 2.7}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "                                                 \n",
+      "100%|██████████| 666/666 [02:15<00:00,  5.71it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'eval_loss': 0.32806867361068726, 'eval_accuracy': 0.858681875792142, 'eval_f1': 0.909312728751525, 'eval_runtime': 1.5521, 'eval_samples_per_second': 1016.663, 'eval_steps_per_second': 8.376, 'epoch': 3.0}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 666/666 [02:16<00:00,  4.89it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'train_runtime': 136.3027, 'train_samples_per_second': 312.518, 'train_steps_per_second': 4.886, 'train_loss': 0.34142845314186254, 'epoch': 3.0}\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "TrainOutput(global_step=666, training_loss=0.34142845314186254, metrics={'train_runtime': 136.3027, 'train_samples_per_second': 312.518, 'train_steps_per_second': 4.886, 'train_loss': 0.34142845314186254, 'epoch': 3.0})"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step10 模型评估"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 13/13 [00:01<00:00,  9.46it/s]\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "{'eval_loss': 0.32806867361068726,\n",
+       " 'eval_accuracy': 0.858681875792142,\n",
+       " 'eval_f1': 0.909312728751525,\n",
+       " 'eval_runtime': 1.513,\n",
+       " 'eval_samples_per_second': 1042.948,\n",
+       " 'eval_steps_per_second': 8.592,\n",
+       " 'epoch': 3.0}"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trainer.evaluate(tokenized_dataset[\"test\"])\n",
+    "#模型预测\n",
+    "#trainer.predict(tokenized_dataset[\"test\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# step11 模型使用"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'label': '好评！', 'score': 0.9663025736808777}]"
+      ]
+     },
+     "execution_count": 13,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "sen = \"这本书真不错！\"\n",
+    "id2_label = {0: \"差评！\", 1: \"好评！\"}\n",
+    "model.eval()\n",
+    "model.config.id2label = id2_label\n",
+    "\n",
+    "pipe = pipeline(\"text-classification\", model=model, tokenizer=tokenizer, device=0)\n",
+    "pipe(sen)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "gitcode",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

命名实体识别.ipynb

@@ -0,0 +1,565 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 命名实体识别"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0.5\n",
+      "              precision    recall  f1-score   support\n",
+      "\n",
+      "        MISC       0.00      0.00      0.00         1\n",
+      "         PER       1.00      1.00      1.00         1\n",
+      "\n",
+      "   micro avg       0.50      0.50      0.50         2\n",
+      "   macro avg       0.50      0.50      0.50         2\n",
+      "weighted avg       0.50      0.50      0.50         2\n",
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from seqeval.metrics import accuracy_score\n",
+    "from seqeval.metrics import classification_report\n",
+    "from seqeval.metrics import f1_score\n",
+    "y_true = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n",
+    "y_pred = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n",
+    "print(f1_score(y_true, y_pred))\n",
+    "print(classification_report(y_true, y_pred))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "d:\\gitcode\\.venv\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "#1导入相关包\n",
+    "import evaluate\n",
+    "from datasets import load_dataset\n",
+    "from transformers import AutoTokenizer, AutoModelForTokenClassification, TrainingArguments, Trainer, DataCollatorForTokenClassification"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Generating train split: 100%|██████████| 20865/20865 [00:00<00:00, 205845.42 examples/s]\n",
+      "Generating validation split: 100%|██████████| 2319/2319 [00:00<00:00, 278085.34 examples/s]\n",
+      "Generating test split: 100%|██████████| 4637/4637 [00:00<00:00, 343925.51 examples/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "#2加载数据集\n",
+    "# 直接使用load_dataset进行加载\n",
+    "ner_datasets = load_dataset(\"D:/gitcode/data/peoples-daily-ner\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'id': '0',\n",
+       " 'tokens': ['海',\n",
+       "  '钓',\n",
+       "  '比',\n",
+       "  '赛',\n",
+       "  '地',\n",
+       "  '点',\n",
+       "  '在',\n",
+       "  '厦',\n",
+       "  '门',\n",
+       "  '与',\n",
+       "  '金',\n",
+       "  '门',\n",
+       "  '之',\n",
+       "  '间',\n",
+       "  '的',\n",
+       "  '海',\n",
+       "  '域',\n",
+       "  '。'],\n",
+       " 'ner_tags': [0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0]}"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ner_datasets[\"train\"][0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'id': Value(dtype='string', id=None),\n",
+       " 'tokens': Sequence(feature=Value(dtype='string', id=None), length=-1, id=None),\n",
+       " 'ner_tags': Sequence(feature=ClassLabel(names=['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC'], id=None), length=-1, id=None)}"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "ner_datasets[\"train\"].features"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "['O', 'B-PER', 'I-PER', 'B-ORG', 'I-ORG', 'B-LOC', 'I-LOC']"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "label_list = ner_datasets[\"train\"].features[\"ner_tags\"].feature.names\n",
+    "label_list"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#3数据预处理\n",
+    "tokenizer = AutoTokenizer.from_pretrained(\"D:/gitcode/model/chinese-macbert-base\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [[101, 3862, 102], [101, 7157, 102], [101, 3683, 102], [101, 6612, 102], [101, 1765, 102], [101, 4157, 102], [101, 1762, 102], [101, 1336, 102], [101, 7305, 102], [101, 680, 102], [101, 7032, 102], [101, 7305, 102], [101, 722, 102], [101, 7313, 102], [101, 4638, 102], [101, 3862, 102], [101, 1818, 102], [101, 511, 102]], 'token_type_ids': [[0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0], [0, 0, 0]], 'attention_mask': [[1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1], [1, 1, 1]]}"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenizer(ner_datasets[\"train\"][0][\"tokens\"]) "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [101, 3862, 7157, 3683, 6612, 1765, 4157, 1762, 1336, 7305, 680, 7032, 7305, 722, 7313, 4638, 3862, 1818, 511, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# 对于已经做好tokenize的数据，要指定is_split_into_words参数为True\n",
+    "tokenizer(ner_datasets[\"train\"][0][\"tokens\"], is_split_into_words=True)   "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'input_ids': [101, 10673, 12865, 12921, 8181, 8681, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1]}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "#有可能一个英文的词，拆成几个token\n",
+    "#所以要对标签做一下额外的处理\n",
+    "res = tokenizer(\"interesting word\")\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[None, 0, 0, 0, 0, 1, None]"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "#若一个单词被拆成了多个token\n",
+    "#如果这些token来自于同一个词,那么这些对应id是相同的\n",
+    "res.word_ids()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 借助word_ids 实现标签映射\n",
+    "def process_function(examples):\n",
+    "    tokenized_exmaples = tokenizer(examples[\"tokens\"],\n",
+    "                                   max_length=128, \n",
+    "                                   truncation=True, \n",
+    "                                   is_split_into_words=True)\n",
+    "    #return tokenized_exmaples\n",
+    "    labels = []\n",
+    "    for i, label in enumerate(examples[\"ner_tags\"]):\n",
+    "        word_ids = tokenized_exmaples.word_ids(batch_index=i)\n",
+    "        label_ids = []\n",
+    "        for word_id in word_ids:\n",
+    "            if word_id is None:\n",
+    "                #CLS SEP 不参与计算\n",
+    "                #在cross loss 中-100不参与运算\n",
+    "                label_ids.append(-100)\n",
+    "            else:\n",
+    "                label_ids.append(label[word_id])\n",
+    "        labels.append(label_ids)\n",
+    "    tokenized_exmaples[\"labels\"] = labels\n",
+    "    return tokenized_exmaples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Map: 100%|██████████| 20865/20865 [00:02<00:00, 9362.95 examples/s] \n",
+      "Map: 100%|██████████| 2319/2319 [00:00<00:00, 8915.19 examples/s] \n",
+      "Map: 100%|██████████| 4637/4637 [00:00<00:00, 9948.85 examples/s] \n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetDict({\n",
+       "    train: Dataset({\n",
+       "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
+       "        num_rows: 20865\n",
+       "    })\n",
+       "    validation: Dataset({\n",
+       "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
+       "        num_rows: 2319\n",
+       "    })\n",
+       "    test: Dataset({\n",
+       "        features: ['id', 'tokens', 'ner_tags', 'input_ids', 'token_type_ids', 'attention_mask', 'labels'],\n",
+       "        num_rows: 4637\n",
+       "    })\n",
+       "})"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tokenized_datasets = ner_datasets.map(process_function, batched=True)\n",
+    "tokenized_datasets"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'id': '0', 'tokens': ['海', '钓', '比', '赛', '地', '点', '在', '厦', '门', '与', '金', '门', '之', '间', '的', '海', '域', '。'], 'ner_tags': [0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0], 'input_ids': [101, 3862, 7157, 3683, 6612, 1765, 4157, 1762, 1336, 7305, 680, 7032, 7305, 722, 7313, 4638, 3862, 1818, 511, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': [-100, 0, 0, 0, 0, 0, 0, 0, 5, 6, 0, 5, 6, 0, 0, 0, 0, 0, 0, -100]}\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(tokenized_datasets[\"train\"][0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at /teacher_train/model/chinese-macbert-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
+      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+     ]
+    }
+   ],
+   "source": [
+    "#4创建模型\n",
+    "# 对于所有的非二分类任务，切记要指定num_labels，否则就会device错误\n",
+    "model = AutoModelForTokenClassification.from_pretrained(\"/teacher_train/model/chinese-macbert-base\",\n",
+    "                                                        num_labels=len(label_list))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "7"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "model.config.num_labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#5创建评估函数\n",
+    "# 这里方便大家加载，替换成了本地的加载方式，无需额外下载\n",
+    "seqeval = evaluate.load(\"seqeval\")\n",
+    "seqeval"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    " # predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n",
+    " # references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]\n",
+    "\n",
+    "def eval_metric(pred):\n",
+    "    predictions, labels = pred\n",
+    "    predictions = np.argmax(predictions, axis=-1)\n",
+    "\n",
+    "    # 将id转换为原始的字符串类型的标签\n",
+    "    true_predictions = [\n",
+    "        [label_list[p] for p, l in zip(prediction, label) if l != -100]\n",
+    "        for prediction, label in zip(predictions, labels) \n",
+    "    ]\n",
+    "\n",
+    "    true_labels = [\n",
+    "        [label_list[l] for p, l in zip(prediction, label) if l != -100]\n",
+    "        for prediction, label in zip(predictions, labels) \n",
+    "    ]\n",
+    "\n",
+    "    result = seqeval.compute(predictions=true_predictions, references=true_labels, scheme=\"IOB2\")\n",
+    "\n",
+    "    return {\n",
+    "        \"f1\": result[\"overall_f1\"]\n",
+    "    }\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "args = TrainingArguments(\n",
+    "    output_dir=\"models_for_ner\",\n",
+    "    logging_dir=\"./logs\",\n",
+    "    per_device_train_batch_size=64,\n",
+    "    per_device_eval_batch_size=128,\n",
+    "    eval_strategy=\"epoch\",\n",
+    "    save_strategy=\"epoch\",\n",
+    "    metric_for_best_model=\"f1\",\n",
+    "    load_best_model_at_end=True,\n",
+    "    logging_steps=50,\n",
+    "    num_train_epochs=1\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/tmp/ipykernel_3129756/746877450.py:9: FutureWarning: `tokenizer` is deprecated and will be removed in version 5.0.0 for `Trainer.__init__`. Use `processing_class` instead.\n",
+      "  trainer = Trainer(\n"
+     ]
+    }
+   ],
+   "source": [
+    "#7创建训练器\n",
+    "\"\"\"\n",
+    "DataCollatorForTokenClassification类适用于序列标注任务（token classification），\n",
+    "比如命名实体识别（NER），每个token都会对应一个预测label，\n",
+    "当一个序列的token比label多时，对额外的label位置添加-100处理，\n",
+    "使得在计算交叉熵损失时，-100位置的label损失为0不用考虑。\n",
+    "使得所有序列保持同一长度，同时对token多于label的场景使用-100进行填充。\n",
+    "\"\"\"\n",
+    "trainer = Trainer(\n",
+    "    model=model,\n",
+    "    args=args,\n",
+    "    tokenizer=tokenizer,\n",
+    "    train_dataset=tokenized_datasets[\"train\"],\n",
+    "    eval_dataset=tokenized_datasets[\"validation\"],\n",
+    "    compute_metrics=eval_metric,\n",
+    "    data_collator=DataCollatorForTokenClassification(tokenizer=tokenizer)\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#8模型训练\n",
+    "trainer.train()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#9评估\n",
+    "trainer.evaluate(eval_dataset=tokenized_datasets[\"test\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#10模型预测\n",
+    "from transformers import pipeline\n",
+    "# 使用pipeline进行推理，要指定id2label\n",
+    "model.config.id2label = {idx: label for idx, label in enumerate(label_list)}\n",
+    "model.config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# 如果模型是基于GPU训练的，那么推理时要指定device\n",
+    "# 对于NER任务，可以指定aggregation_strategy为simple，得到具体的实体的结果，而不是token的结果\n",
+    "ner_pipe = pipeline(\"token-classification\", model=model, tokenizer=tokenizer, device=0, aggregation_strategy=\"simple\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "gitcode",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}