MOE<\/code> \u6df7\u5408\u4e13\u5bb6\u67b6\u6784\u6240\u4f7f\u7528\u7684\u8bad\u7ec3\u6570\u636e\u96c6\u548c\u8bcd\u8868\u5c31\u4e0d\u518d\u91cd\u590d\u8bf4\u660e\uff0c\u76f4\u63a5\u90fd\u590d\u7528\u4e0a\u7bc7\u6587\u7ae0\u7684\u5185\u5bb9\u3002<\/p>\n\u8fd8\u6709\u5bf9\u4e8e\u7ec6\u8282\u7684 \u70b9\u79ef\u6ce8\u610f\u529b\u5c42\u3001\u591a\u5934\u6ce8\u610f\u529b\u5c42\u3001\u5012\u4e09\u89d2\u63a9\u7801\u5668\u3001\u4f4d\u7f6e\u7f16\u7801 \u7b49\u7b49\u7684\u8ba1\u7b97\u8fc7\u7a0b\u548c\u516c\u5f0f\u4e5f\u90fd\u8bf7\u53c2\u8003\u4e0a\u7bc7\u6587\u7ae0\u4e2d\u7684\u4ecb\u7ecd\uff0c\u672c\u7bc7\u5185\u5bb9\u6700\u540e\u5b9e\u73b0\u7684\u6548\u679c\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n
![\"\u56fe\u7247\"](\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251107135249701.png\")
<\/p>\n
\u5b9e\u9a8c\u6240\u4f7f\u7528\u7684\u4e3b\u8981\u4f9d\u8d56\u7248\u672c\u5982\u4e0b\uff1a<\/p>\n
torch==2.6.0\ntensorboard==2.19.0<\/code><\/pre>\n\u4e8c\u3001\u642d\u5efa GPTMoEModel \u7f51\u7edc\u67b6\u6784<\/h2>\n2.1 \u5b9e\u73b0\uff08\u70b9\u79ef\u8ba1\u7b97\u3001\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236 \uff09<\/h3>\n
\u70b9\u79ef\u8ba1\u7b97\u3001\u591a\u5934\u6ce8\u610f\u529b\u673a\u5236 \u5b9e\u73b0\u903b\u8f91\u548c\u4e0a\u7bc7\u6587\u7ae0\u4e2d\u4e00\u81f4\uff0c\u5982\u4e0b\u6240\u793a\uff0c\u5176\u4e2d\u5173\u952e\u90e8\u5206\u90fd\u505a\u4e86\u6ce8\u91ca\u8bf4\u660e\uff1a<\/p>\n
import torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.distributions.normal import Normal\nimport numpy as np\n\n# \u70b9\u79ef\u8ba1\u7b97\nclass ScaledDotProductAttention(nn.Module):\n def __init__(self, d_k):\n super(ScaledDotProductAttention, self).__init__()\n self.d_k = d_k\n\n def forward(self, q, k, v, attention_mask):\n ##\n # q: [batch_size, n_heads, len_q, d_k]\n # k: [batch_size, n_heads, len_k, d_k]\n # v: [batch_size, n_heads, len_v, d_v]\n # attn_mask: [batch_size, n_heads, seq_len, seq_len]\n ##\n # \u8ba1\u7b97\u6bcf\u4e2aQ\u4e0eK\u7684\u5206\u6570\uff0c\u8ba1\u7b97\u51fa\u6765\u7684\u5927\u5c0f\u662f [batch_size, n_heads, len_q, len_q]\n scores = torch.matmul(q, k.transpose(-1, -2)) \/ np.sqrt(self.d_k)\n # \u628a\u88abmask\u7684\u5730\u65b9\u7f6e\u4e3a\u65e0\u9650\u5c0f\uff0csoftmax\u4e4b\u540e\u57fa\u672c\u5c31\u662f0\uff0c\u4e5f\u5c31\u5bf9q\u4e0d\u8d77\u4f5c\u7528\n scores.masked_fill_(attention_mask, -1e9)\n attn = nn.Softmax(dim=-1)(scores)\n # \u6ce8\u610f\u529b\u540e\u7684\u5927\u5c0f [batch_size, n_heads, len_q, d_v]\n context = torch.matmul(attn, v)\n return context, attn\n\n# \u591a\u5934\u6ce8\u610f\u529b\u673a\u5236\nclass MultiHeadAttention(nn.Module):\n def __init__(self, d_model, n_heads, d_k, d_v):\n super(MultiHeadAttention, self).__init__()\n self.d_model = d_model\n self.n_heads = n_heads\n self.d_k = d_k\n self.d_v = d_v\n self.w_q = nn.Linear(d_model, d_k * n_heads, bias=False)\n self.w_k = nn.Linear(d_model, d_k * n_heads, bias=False)\n self.w_v = nn.Linear(d_model, d_v * n_heads, bias=False)\n self.fc = nn.Linear(n_heads * d_v, d_model, bias=False)\n self.layernorm = nn.LayerNorm(d_model)\n\n def forward(self, q, k, v, attention_mask):\n ##\n # q: [batch_size, seq_len, d_model]\n # k: [batch_size, seq_len, d_model]\n # v: [batch_size, seq_len, d_model]\n # attn_mask: [batch_size, seq_len, seq_len]\n ##\n # \u8bb0\u5f55\u539f\u59cb\u503c, \u540e\u7eed\u8ba1\u7b97\u6b8b\u5dee\n residual, batch_size = q, q.size(0)\n # \u5148\u6620\u5c04 q\u3001k\u3001v, \u7136\u540e\u540e\u5206\u5934\uff1b\n # q: [batch_size, n_heads, len_q, d_k]\n q = self.w_q(q).view(batch_size, -1, self.n_heads, self.d_k).transpose(1, 2)\n # k: [batch_size, n_heads, len_k, d_k]\n k = self.w_k(k).view(batch_size, -1, self.n_heads, self.d_k).transpose(1, 2)\n # v: [batch_size, n_heads, len_v(=len_k), d_v]\n v = self.w_v(v).view(batch_size, -1, self.n_heads, self.d_v).transpose(1, 2)\n # attn_mask : [batch_size, n_heads, seq_len, seq_len]\n attention_mask = attention_mask.unsqueeze(1).repeat(1, self.n_heads, 1, 1)\n # \u70b9\u79ef\u6ce8\u610f\u529b\u5206\u6570\u8ba1\u7b97\uff0c [batch_size, n_heads, len_q, d_v]\n context, attn = ScaledDotProductAttention(self.d_k)(q, k, v, attention_mask)\n # context: [batch_size, len_q, n_heads * d_v]\n context = context.transpose(1, 2).reshape(batch_size, -1, self.n_heads * self.d_v)\n # \u8fd8\u539f\u4e3a\u539f\u59cb\u5927\u5c0f\n output = self.fc(context)\n # LN + \u6b8b\u5dee\u8ba1\u7b97\n return self.layernorm(output + residual), attn\n<\/code><\/pre>\n2.2 \u5b9e\u73b0\u95e8\u63a7\u7f51\u7edcRouter<\/h3>\n
\u95e8\u63a7\u7f51\u7edc\u5c31\u662f\u4e00\u4e2a\u8f7b\u91cf\u7ea7\u7684\u795e\u7ecf\u7f51\u7edc\uff0c\u5b83\u7684\u4f5c\u7528\uff1a\u5bf9\u6bcf\u4e00\u4e2a token<\/code>\uff0c\u9884\u6d4b\u5176\u5e94\u88ab\u5206\u914d\u7ed9\u54ea\u4e9b\u4e13\u5bb6\uff0c\u5e76\u4e3a\u6bcf\u4e2a\u9009\u4e2d\u7684\u4e13\u5bb6\u5206\u914d\u4e00\u4e2a\u6743\u91cd\uff0c\u7528\u4e8e\u52a0\u6743\u878d\u5408\u591a\u4e2a\u4e13\u5bb6\u7684\u8f93\u51fa\u3002<\/p>\n\u4f46\u662f\u95e8\u63a7\u7f51\u7edc\u6709\u4e2a\u95ee\u9898\u5c31\u662f\u53ef\u80fd\u4f1a\u53d1\u751f \u4e13\u5bb6\u5931\u8861 \uff0c\u603b\u662f\u5c06\u6837\u672c\u5206\u914d\u7ed9\u5c11\u6570\u51e0\u4e2a\u80fd\u529b\u5f3a\u6216\u521d\u59cb\u5316\u7684\u597d\u7684\u4e13\u5bb6\uff0c\u5bfc\u81f4\u5176\u4ed6\u4e13\u5bb6\u5f97\u4e0d\u5230\u8bad\u7ec3\uff0c\u6700\u7ec8\u6574\u4e2a\u7cfb\u7edf\u9000\u5316\uff0c\u53ea\u6709\u5c11\u6570\u4e13\u5bb6\u88ab\u4f7f\u7528\u3002\u4e3a\u4e86\u89e3\u51b3\u8fd9\u4e2a\u95ee\u9898\uff0c\u53ef\u4ee5\u5728\u8def\u7531\u65f6\uff0c\u589e\u52a0\u4e00\u4e2a\u53ef\u8bad\u7ec3\u7684\u566a\u58f0\uff0c\u53e6\u5916\u8fd8\u9700\u8981\u5f15\u5165\u4e00\u4e2a\u8f85\u52a9\u635f\u5931\uff0c\u4e5f\u5c31\u662f\u8d1f\u8f7d\u5747\u8861\u635f\u5931\uff0c\u8fd9\u91cc\u8d1f\u8f7d\u5747\u8861\u635f\u5931\u53c2\u8003 Mixtral<\/code> \u6a21\u578b\u7684\u505a\u6cd5\u3002<\/p>\n\u5b9e\u73b0\u903b\u8f91\u5982\u4e0b\uff1a<\/p>\n
# \u95e8\u63a7\u7f51\u7edc\nclass Router(nn.Module):\n def __init__(self, d_model, num_experts, top_k=2):\n super(Router, self).__init__()\n self.num_experts = num_experts\n self.top_k = top_k\n self.gate = nn.Linear(d_model, num_experts)\n # \u7528\u4e8e\u8d1f\u8f7d\u5747\u8861\u7684\u566a\u58f0\n self.noise_linear = nn.Linear(d_model, num_experts)\n\n def forward(self, x):\n logits = self.gate(x)\n\n # \u8bad\u7ec3\u65f6\u6dfb\u52a0\u566a\u58f0\n if self.training:\n noise = torch.randn_like(logits).to(x.device)\n noise = self.noise_linear(x) * noise\n noisy_logits = logits + noise\n else:\n noisy_logits = logits\n\n gates_prob = F.softmax(noisy_logits, dim=-1)\n # Top-k \u9009\u62e9\n top_k_probs, top_k_indices = torch.topk(gates_prob, self.top_k, dim=-1)\n # \u5f52\u4e00\u5316\uff0c\u786e\u4fdd\u88ab\u9009\u4e2d\u7684\u4e13\u5bb6\u7684\u6743\u91cd\u4e4b\u548c\u4e3a1\n top_k_probs = top_k_probs \/ top_k_probs.sum(dim=-1, keepdim=True)\n # \u8d1f\u8f7d\u5747\u8861\u635f\u5931\n load_balancing_loss = self.compute_load_balancing_loss(gates_prob, top_k_indices)\n return top_k_probs, top_k_indices, load_balancing_loss\n\n def compute_load_balancing_loss(self, gates_prob, top_k_indices):\n """ \u8d1f\u8f7d\u5747\u8861\u635f\u5931\uff1anum_experts * sum ( \u6bcf\u4e2a\u4e13\u5bb6\u7684\u5e73\u5747\u6982\u7387 * \u6bcf\u4e2a\u4e13\u5bb6\u9009\u4e2d\u7684\u6982\u7387 )"""\n batch_size, seq_len, _ = gates_prob.shape\n\n # \u8ba1\u7b97\u6bcf\u4e2a\u4e13\u5bb6\u7684\u5e73\u5747\u6982\u7387\n router_prob_per_expert = gates_prob.mean(dim=(0, 1))\n\n # \u8ba1\u7b97\u6bcf\u4e2a\u4e13\u5bb6\u7406\u60f3\u88ab\u5206\u914d\u5230\u7684\u6982\u7387\n expert_mask = torch.zeros_like(gates_prob)\n expert_mask.scatter_(2, top_k_indices, 1)\n tokens_per_expert = expert_mask.float().mean(dim=(0, 1))\n\n # \u8f85\u52a9\u635f\u5931\n return self.num_experts * torch.sum(tokens_per_expert * router_prob_per_expert)<\/code><\/pre>\n2.3 \u5b9e\u73b0\u4e13\u5bb6\u7f51\u7edc<\/h3>\n
\u6bcf\u4e2a\u4e13\u5bb6\u76f8\u5f53\u4e8e\u662f\u4e00\u4e2a\u524d\u9988\u795e\u7ecf\u7f51\u7edc\uff0c \u8fd9\u91cc\u6a21\u62dfSwiGLU FFN<\/code>\u3002<\/p>\n# \u4e13\u5bb6\u7f51\u7edc\nclass Expert(nn.Module):\n def __init__(self, d_model, d_ff):\n super(Expert, self).__init__()\n self.w1 = nn.Linear(d_model, d_ff, bias=False)\n self.w2 = nn.Linear(d_model, d_ff, bias=False)\n self.w_out = nn.Linear(d_ff, d_model, bias=False)\n\n def forward(self, x):\n return self.w_out(F.silu(self.w1(x)) * self.w2(x))<\/code><\/pre>\n2.4 \u6574\u5408Router\u548c\u4e13\u5bb6\u5c42\uff0c\u5b9e\u73b0 MOE \u5c42<\/h3>\n
\u5305\u62ec\u4e00\u4e2a \u95e8\u63a7Router<\/code>\uff0c\u548c\u591a\u4e2a\u4e13\u5bb6\u7ec4\u6210\u3002Router<\/code> \u8f93\u51fa top-k<\/code> \u4e13\u5bb6 ID<\/code> \u548c\u6743\u91cd\uff0c\u7136\u540e\u5c06 token<\/code> \u8f93\u5165\u5230\u5bf9\u5e94\u4e13\u5bb6\uff1b\u7136\u540e\u52a0\u6743\u878d\u5408\u8f93\u51fa<\/p>\n\u8fd9\u91cc\u4e3a\u4e86\u53ef\u4ee5\u66f4\u52a0\u5229\u4e8e\u7406\u89e3\uff0c\u5728\u505a\u4e13\u5bb6\u9009\u62e9\u65f6\uff0c\u7528\u7684\u53cc\u91cd\u5faa\u73af + \u9010\u4e13\u5bb6\u5224\u65ad\uff0c\u53ef\u80fd\u65e0\u6cd5\u9ad8\u6548\u7684\u5229\u7528GPU<\/code>\u7684\u5e76\u884c\u8ba1\u7b97\uff0c\u8fd9\u91cc\u53ef\u4ee5\u53c2\u8003 Mixtral<\/code> \u6a21\u578b\u7684\u5199\u6cd5\u66f4\u9ad8\u6548\u7684\u8fd0\u884c\u3002<\/p>\n# MOE\u5c42\nclass MoELayer(nn.Module):\n def __init__(self, d_model, d_ff, num_experts=8, top_k=2):\n super(MoELayer, self).__init__()\n self.d_model = d_model\n self.num_experts = num_experts\n self.top_k = top_k\n # \u95e8\u63a7\u8def\u7531\uff0c\u51b3\u5b9a\u54ea\u4e9b\u4e13\u5bb6\u88ab\u6fc0\u6d3b\n self.router = Router(d_model, num_experts, top_k)\n # \u521b\u5efa\u591a\u4e2a\u4e13\u5bb6\n self.experts = nn.ModuleList([\n Expert(d_model, d_ff) for _ in range(num_experts)\n ])\n # Layer Norm\n self.layernorm = nn.LayerNorm(d_model)\n\n def forward(self, x):\n """\n x: [batch_size, seq_len, d_model]\n """\n residual = x\n\n batch_size, seq_len, d_model = x.shape\n # \u83b7\u53d6\u8def\u7531\u51b3\u7b56\n # gates: [batch_size, seq_len, top_k]\n # selected_experts: [batch_size, seq_len, top_k]\n gates, selected_experts, load_balancing_loss = self.router(x)\n\n # \u521d\u59cb\u5316\u8f93\u51fa\n output = torch.zeros_like(x)\n\n # \u5bf9\u6bcf\u4e2atoken\u5e94\u7528\u9009\u4e2d\u7684\u4e13\u5bb6\n for i in range(self.top_k):\n # \u83b7\u53d6\u5f53\u524d\u4e13\u5bb6\u7d22\u5f15\n expert_idx = selected_experts[:, :, i] # [batch_size, seq_len]\n # \u83b7\u53d6\u5f53\u524d\u6743\u91cd\n expert_gate = gates[:, :, i] # [batch_size, seq_len]\n # \u5bf9\u6bcf\u4e2a\u4e13\u5bb6\u8fdb\u884c\u8ba1\u7b97\n for expert_id in range(self.num_experts):\n # \u627e\u51fa\u9009\u62e9\u4e86\u5f53\u524d\u4e13\u5bb6\u7684token\u4f4d\u7f6e\n mask = (expert_idx == expert_id).unsqueeze(-1) # [batch_size, seq_len, 1]\n if mask.any():\n # \u83b7\u53d6\u5206\u914d\u7ed9\u5f53\u524d\u4e13\u5bb6\u7684tokens\n expert_input = x * mask # [batch_size, seq_len, d_model]\n # \u5e94\u7528\u4e13\u5bb6\n expert_output = self.experts[expert_id](expert_input) # [batch_size, seq_len, d_model]\n # \u52a0\u6743\u8f93\u51fa\n weighted_output = expert_output * expert_gate.unsqueeze(-1) * mask\n output += weighted_output\n # \u6b8b\u5dee\u8fde\u63a5\u548cLayer Norm\n output = self.layernorm(output + residual)\n return output, load_balancing_loss<\/code><\/pre>\n2.5 \u5b9e\u73b0 MOE \u89e3\u7801\u5c42<\/h3>\n
\u548c\u4f20\u7edf\u7684 Transformer Decoder Layer<\/code> \u7c7b\u4f3c\uff0c\u53ea\u9700\u5c06 \u524d\u9988\u7f51\u7edcFFN<\/code> \u6362\u6210 MOE<\/code> \u5c42\u3002<\/p>\n# \u89e3\u7801\u5c42\nclass MoEDecoderLayer(nn.Module):\n def __init__(self, d_model, n_heads, d_ff, d_k, d_v, num_experts=8, top_k=2):\n super(MoEDecoderLayer, self).__init__()\n # \u591a\u5934\u6ce8\u610f\u529b\u5c42\n self.attention = MultiHeadAttention(d_model, n_heads, d_k, d_v)\n # MoE\n self.pos_ffn = MoELayer(d_model, d_ff, num_experts, top_k)\n\n def forward(self, inputs, attention_mask):\n # \u591a\u5934\u6ce8\u610f\u529b\n outputs, self_attn = self.attention(inputs, inputs, inputs, attention_mask)\n # MoE\n outputs, load_balancing_loss = self.pos_ffn(outputs)\n return outputs, self_attn, load_balancing_loss<\/code><\/pre>\n2.6 \u5806\u79efMOE\u89e3\u7801\u5c42\uff0c\u5b9e\u73b0 MOE \u89e3\u7801\u5668<\/h3>\n
\u5c06\u591a\u4e2a\u89e3\u7801\u5c42\u5806\u53e0\uff0c\u5f62\u6210\u4e00\u4e2a\u7279\u5f81\u63d0\u53d6\u94fe\u3002\u4e3a\u4e86\u4fbf\u4e8e\u548c\u4e0a\u7bc7\u6587\u7ae0\u505a\u6548\u679c\u5bf9\u6bd4\uff0c\u8fd9\u91cc\u4f4d\u7f6e\u7f16\u7801\u4f9d\u7136\u4f7f\u7528 GPT2<\/code> \u7684\u505a\u6cd5\uff0c\u540c\u6837\u4e5f\u9700\u8981\u4e00\u4e2a\u5012\u4e09\u89d2\u63a9\u7801\u5668\uff0c\u9632\u6b62\u6a21\u578b\u770b\u5230\u672a\u6765\u7684\u4fe1\u606f\u3002<\/p>\n\u63a9\u7801\u8fc7\u7a0b\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n
\u539f\u59cb\u6ce8\u610f\u529b\u5206\u6570\u77e9\u9635\uff08\u65e0\u63a9\u7801\uff09:\n[[q1k1, q1k2, q1k3, q1k4],\n [q2k1, q2k2, q3k3, q3k4],\n [q3k1, q3k2, q3k3, q3k4],\n [q4k1, q4k2, q4k3, q4k4]]\n\n\u4e0a\u4e09\u89d2\u63a9\u7801\u5668:\n[[0, 1, 1, 1],\n [0, 0, 1, 1],\n [0, 0, 0, 1],\n [0, 0, 0, 0]]\n\n\u5e94\u7528\u63a9\u7801\u540e\u7684\u5206\u6570\u77e9\u9635:\n[[q1k1, -inf, -inf, -inf],\n [q2k1, q2k2, -inf, -inf],\n [q3k1, q3k2, q3k3, -inf],\n [q4k1, q4k2, q4k3, q4k4]]<\/code><\/pre>\n\u5b9e\u73b0\u903b\u8f91\u5982\u4e0b\uff1a<\/p>\n
# \u4f4d\u7f6e\u7f16\u7801\uff0c\u8fd9\u91cc\u4f7f\u7528GPT2\u7684\u505a\u6cd5\nclass PositionalEncoding(nn.Module):\n def __init__(self, d_model, max_pos, device):\n super(PositionalEncoding, self).__init__()\n self.device = device\n self.pos_embedding = nn.Embedding(max_pos, d_model)\n\n def forward(self, inputs):\n seq_len = inputs.size(1)\n pos = torch.arange(seq_len, dtype=torch.long, device=self.device)\n # [seq_len] -> [batch_size, seq_len]\n pos = pos.unsqueeze(0).expand_as(inputs)\n return self.pos_embedding(pos)\n\n# \u83b7\u53d6pad\u63a9\u7801\u5668\ndef get_attn_pad_mask(attention_mask):\n batch_size, len_seq = attention_mask.size()\n attention_mask = attention_mask.data.eq(0).unsqueeze(1)\n # \u6ce8\u610f\u529b\u5206\u6570\u7684\u5927\u5c0f\u662f [batch_size, n_heads, len_q, len_q]\n # \u6240\u4ee5\u8fd9\u91cc\u8981\u8f6c\u6362\u6210 [batch_size, len_seq, len_seq] \u5927\u5c0f\n return attention_mask.expand(batch_size, len_seq, len_seq)\n\n# \u83b7\u53d6\u5012\u4e09\u89d2\u63a9\u7801\u5668\uff0c\u9632\u6b62\u6a21\u578b\u770b\u5230\u672a\u6765\u7684\u4fe1\u606f\ndef get_attn_subsequence_mask(seq, device):\n # \u6ce8\u610f\u529b\u5206\u6570\u7684\u5927\u5c0f\u662f [batch_size, n_heads, len_seq, len_seq]\n # \u6240\u4ee5\u8fd9\u91cc\u8981\u751f\u6210 [batch_size, len_seq, len_seq] \u5927\u5c0f\n attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n # \u751f\u6210\u4e00\u4e2a\u4e0a\u4e09\u89d2\u77e9\u9635\n subsequence_mask = np.triu(np.ones(attn_shape), k=1)\n subsequence_mask = torch.from_numpy(subsequence_mask).byte()\n subsequence_mask = subsequence_mask.to(device)\n return subsequence_mask\n\n# \u89e3\u7801\u5668\nclass MoEDecoder(nn.Module):\n def __init__(self, d_model, n_heads, d_ff, d_k, d_v, vocab_size, max_pos, n_layers,\n device, num_experts=8, top_k=2):\n super(MoEDecoder, self).__init__()\n self.device = device\n # \u5c06Token\u8f6c\u4e3a\u5411\u91cf\n self.embedding = nn.Embedding(vocab_size, d_model)\n # \u4f4d\u7f6e\u7f16\u7801\n self.pos_encoding = PositionalEncoding(d_model, max_pos, device)\n\n # \u521b\u5efaMOE\u5c42\n self.layers = nn.ModuleList()\n for i in range(n_layers):\n self.layers.append(\n MoEDecoderLayer(\n d_model, n_heads, d_ff, d_k, d_v,\n num_experts, top_k\n )\n )\n\n def forward(self, inputs, attention_mask):\n # \u5d4c\u5165\u548c\u4f4d\u7f6e\u7f16\u7801\n outputs = self.embedding(inputs) + self.pos_encoding(inputs)\n\n # \u751f\u6210\u63a9\u7801\n subsequence_mask = get_attn_subsequence_mask(inputs, self.device)\n if attention_mask is not None:\n attention_mask = get_attn_pad_mask(attention_mask)\n attention_mask = torch.gt((attention_mask + subsequence_mask), 0)\n else:\n attention_mask = subsequence_mask.bool()\n\n # \u8ba1\u7b97\u6bcf\u4e00\u5c42\u7684\u7ed3\u679c\n self_attns = []\n total_load_balancing_loss = 0.0\n for layer in self.layers:\n layer_output = layer(outputs, attention_mask)\n outputs, self_attn, load_balancing_loss = layer_output\n total_load_balancing_loss += load_balancing_loss\n self_attns.append(self_attn)\n\n return outputs, self_attns, total_load_balancing_loss<\/code><\/pre>\n2.7 \u6574\u5408\u89e3\u7801\u5668\uff0c\u5b9e\u73b0 GPTMoEModel<\/h3>\n
\u8fd9\u91cc\u9700\u8981\u6ce8\u610f\uff0c\u635f\u5931\u51fd\u6570\u8981\u8003\u8651\u524d\u9762\u7684\u8d1f\u8f7d\u5747\u8861\u635f\u5931\uff0c\u56e0\u6b64\u6574\u4f53\u7684\u635f\u5931\u5e94\u8be5\u662f\u4e24\u8005\u4e4b\u548c\u3002<\/p>\n
# GPT MOE\u6a21\u578b\nclass GPTMoEModel(nn.Module):\n def __init__(self, d_model, n_heads, d_ff, d_k, d_v, vocab_size, max_pos, n_layers,\n device, num_experts=8, top_k=2, load_balancing_weight=0.01):\n super(GPTMoEModel, self).__init__()\n self.load_balancing_weight = load_balancing_weight\n # \u89e3\u7801\u5668\n self.decoder = MoEDecoder(\n d_model, n_heads, d_ff, d_k, d_v, vocab_size, max_pos, n_layers,\n device, num_experts, top_k\n )\n # \u6620\u5c04\u4e3a\u8bcd\u8868\u5927\u5c0f\n self.projection = nn.Linear(d_model, vocab_size)\n\n def forward(self, inputs, attention_mask=None, targets=None):\n # \u524d\u5411\u4f20\u64ad\n outputs, self_attns, load_balancing_loss = self.decoder(inputs, attention_mask)\n # \u6295\u5f71\u5230\u8bcd\u8868\n logits = self.projection(outputs)\n logits = logits.view(-1, logits.size(-1))\n if targets is not None:\n # \u8d1f\u8f7d\u5747\u8861\u635f\u5931\n load_balancing_loss = load_balancing_loss * self.load_balancing_weight\n # \u4efb\u52a1\u635f\u5931\n lm_loss = F.cross_entropy(logits, targets.view(-1), ignore_index=0)\n # MOE\u67b6\u6784\u7684\u603b\u635f\u5931\u662f\u4efb\u52a1\u635f\u5931\u548c\u8d1f\u8f7d\u5747\u8861\u635f\u5931\u7684\u52a0\u6743\u548c\n total_loss = lm_loss + load_balancing_loss\n return logits, self_attns, total_loss\n return logits, self_attns<\/code><\/pre>\n2.8 \u6574\u4f53\u7f51\u7edc\u67b6\u6784<\/h3>\n
\u4ee5\u4e0a\u6574\u4f53\u7f51\u7edc\u4ee3\u7801\u653e\u5728 model_moe.py<\/code> \u4e2d\u3002<\/p>\nimport torch\nfrom model_moe import GPTMoEModel\n\ndef main():\n device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")\n # \u6a21\u578b\u53c2\u6570\n model_param = {\n "d_model": 768, # \u5d4c\u5165\u5c42\u5927\u5c0f\n "d_ff": 2048, # \u8fd9\u662f\u4e3a\u4e13\u5bb6\u7f51\u7edc\u5927\u5c0f\n "d_k": 64, # K \u7684\u5927\u5c0f\n "d_v": 64, # V \u7684\u5927\u5c0f\n "n_layers": 6, # \u89e3\u7801\u5c42\u7684\u6570\u91cf\n "n_heads": 8, # \u591a\u5934\u6ce8\u610f\u529b\u7684\u5934\u6570\n "max_pos": 1800, # \u4f4d\u7f6e\u7f16\u7801\u7684\u957f\u5ea6\n "device": device, # \u8bbe\u5907\n "vocab_size": 4825, # \u8bcd\u8868\u5927\u5c0f\uff0c\u4e0a\u7bc7\u6587\u7ae0\u4e2d\u6784\u5efa\u7684\u8bcd\u8868\u5927\u5c0f\n "num_experts": 8, # 8\u4e2a\u4e13\u5bb6\n "top_k": 2, # \u6bcf\u4e2atoken\u9009\u62e92\u4e2a\u4e13\u5bb6\n "load_balancing_weight": 0.01 # \u8d1f\u8f7d\u5747\u8861\u635f\u5931\u6743\u91cd\n }\n model = GPTMoEModel(**model_param)\n total_params = sum(p.numel() for p in model.parameters())\n print(model)\n print("total_params: ", total_params)\n\nif __name__ == '__main__':\n main()\nGPTMoEModel(\n (decoder): MoEDecoder(\n (embedding): Embedding(4825, 768)\n (pos_encoding): PositionalEncoding(\n (pos_embedding): Embedding(1800, 768)\n )\n (layers): ModuleList(\n (0-5): 6 x MoEDecoderLayer(\n (attention): MultiHeadAttention(\n (w_q): Linear(in_features=768, out_features=512, bias=False)\n (w_k): Linear(in_features=768, out_features=512, bias=False)\n (w_v): Linear(in_features=768, out_features=512, bias=False)\n (fc): Linear(in_features=512, out_features=768, bias=False)\n (layernorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n )\n (pos_ffn): MoELayer(\n (router): Router(\n (gate): Linear(in_features=768, out_features=8, bias=True)\n (noise_linear): Linear(in_features=768, out_features=8, bias=True)\n )\n (experts): ModuleList(\n (0-7): 8 x Expert(\n (fc1): Linear(in_features=768, out_features=2048, bias=False)\n (fc2): Linear(in_features=2048, out_features=768, bias=False)\n (activation): ReLU()\n )\n )\n (layernorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n )\n )\n )\n )\n (projection): Linear(in_features=768, out_features=4825, bias=True)\n)\ntotal_params: 173028409<\/code><\/pre>\n\u6574\u4f53\u53c2\u6570\u91cf\u4e3a 1.73\u4ebf<\/code> \uff0c 0.17B<\/code> \u5927\u5c0f\uff0c\u76f8\u6bd4\u4e0a\u7bc7\u6587\u7ae0\u6784\u5efa\u7684\u7f51\u7edc\uff0c\u80fd\u5bb9\u7eb3\u66f4\u591a\u7684\u77e5\u8bc6\u3002<\/p>\n\u4e09\u3001\u6a21\u578b\u8bad\u7ec3<\/h2>\n
\u8fd9\u91cc\u8bad\u7ec3\u96c6\u548c\u8bad\u7ec3\u8fc7\u7a0b\u57fa\u672c\u548c\u4e0a\u7bc7\u6587\u7ae0\u4e00\u81f4\uff0c\u540c\u65f6\u8bad\u7ec3\u6570\u636e\u96c6\u4e2d\u540c\u6837\u589e\u52a0\u4e00\u4e9b\u81ea\u5b9a\u4e49\u7684\u6a21\u578b\u7279\u8272\u5185\u5bb9\uff0c\u8ffd\u52a0\u51e0\u6761\u8eab\u4efd\u7684\u6570\u636e\u5728\u91cc\u9762\uff1a<\/p>\n
{"question": "\u4f60\u662f\u8c01", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u53eb\u4ec0\u4e48", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u7684\u540d\u5b57\u662f\u4ec0\u4e48", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u53eb\u5565", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u540d\u5b57\u662f\u5565", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u662f\u4ec0\u4e48\u8eab\u4efd", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u7684\u5168\u540d\u662f\u4ec0\u4e48", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u81ea\u79f0\u4ec0\u4e48", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u7684\u79f0\u53f7\u662f\u4ec0\u4e48", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}\n{"question": "\u4f60\u7684\u6635\u79f0\u662f\u4ec0\u4e48", "answer": "\u6211\u662f\u5c0f\u6bd5\u8d85,\u4e00\u4e2a\u7b80\u6613\u7684\u5c0f\u52a9\u624b"}<\/code><\/pre>\n![\"\u56fe\u7247\"](\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251107135250034.png\")
<\/p>\n
3.1 \u6784\u5efa Dataset<\/h3>\n
qa_dataset.py<\/p>\n
# -*- coding: utf-8 -*-\nfrom torch.utils.data import Dataset\nimport torch\nimport json\nimport numpy as np\n\nclass QADataset(Dataset):\n def __init__(self, data_path, tokenizer, max_length) -> None:\n super().__init__()\n self.tokenizer = tokenizer\n self.max_length = max_length\n self.data = []\n if data_path:\n with open(data_path, "r", encoding='utf-8') as f:\n for line in f:\n if not line or line == "":\n continue\n json_line = json.loads(line)\n question = json_line["question"]\n answer = json_line["answer"]\n self.data.append({\n "question": question,\n "answer": answer\n })\n print("data load \uff0c size\uff1a", len(self.data))\n\n def preprocess(self, question, answer):\n encode, att_mask = self.tokenizer.encode(question, answer, max_length=self.max_length, pad_to_max_length=True)\n input_ids = encode[:-1]\n att_mask = att_mask[:-1]\n labels = encode[1:]\n return input_ids, att_mask, labels\n\n def __getitem__(self, index):\n item_data = self.data[index]\n input_ids, att_mask, labels = self.preprocess(**item_data)\n return {\n "input_ids": torch.LongTensor(np.array(input_ids)),\n "attention_mask": torch.LongTensor(np.array(att_mask)),\n "labels": torch.LongTensor(np.array(labels))\n }\n\n def __len__(self):\n return len(self.data)<\/code><\/pre>\n3.2 \u8bad\u7ec3<\/h3>\n# -*- coding: utf-8 -*-\nimport torch\nfrom torch.utils.data import DataLoader\nfrom torch.utils.tensorboard import SummaryWriter\nfrom tokenizer import Tokenizer\nfrom model_moe import GPTMoEModel\nfrom qa_dataset import QADataset\nfrom tqdm import tqdm\nimport time, sys, os\n\ndef train_model(model, train_loader, val_loader, optimizer,\n device, num_epochs, model_output_dir, writer):\n batch_step = 0\n best_val_loss = float('inf')\n for epoch in range(num_epochs):\n time1 = time.time()\n model.train()\n for index, data in enumerate(tqdm(train_loader, file=sys.stdout, desc="Train Epoch: " + str(epoch))):\n input_ids = data['input_ids'].to(device, dtype=torch.long)\n attention_mask = data['attention_mask'].to(device, dtype=torch.long)\n labels = data['labels'].to(device, dtype=torch.long)\n optimizer.zero_grad()\n outputs, dec_self_attns, loss = model(input_ids, attention_mask, labels)\n loss.backward()\n # \u68af\u5ea6\u88c1\u526a\n torch.nn.utils.clip_grad_norm_(model.parameters(), 1)\n optimizer.step()\n writer.add_scalar('Loss\/train', loss, batch_step)\n batch_step += 1\n # 50\u8f6e\u6253\u5370\u4e00\u6b21 loss\n if index % 50 == 0 or index == len(train_loader) - 1:\n time2 = time.time()\n tqdm.write(\n f"{index}, epoch: {epoch} -loss: {str(loss)} ; lr: {optimizer.param_groups[0]['lr']} ;each step's time spent: {(str(float(time2 - time1) \/ float(index + 0.0001)))}")\n # \u9a8c\u8bc1\n model.eval()\n val_loss = validate_model(model, device, val_loader)\n writer.add_scalar('Loss\/val', val_loss, epoch)\n print(f"val loss: {val_loss} , epoch: {epoch}")\n # \u4fdd\u5b58\u6700\u4f18\u6a21\u578b\n if val_loss < best_val_loss:\n best_val_loss = val_loss\n best_model_path = os.path.join(model_output_dir, "best.pt")\n print("Save Best Model To ", best_model_path, ", epoch: ", epoch)\n torch.save(model.state_dict(), best_model_path)\n # \u4fdd\u5b58\u5f53\u524d\u6a21\u578b\n last_model_path = os.path.join(model_output_dir, "last.pt")\n print("Save Last Model To ", last_model_path, ", epoch: ", epoch)\n torch.save(model.state_dict(), last_model_path)\n\ndef validate_model(model, device, val_loader):\n running_loss = 0.0\n with torch.no_grad():\n for _, data in enumerate(tqdm(val_loader, file=sys.stdout, desc="Validation Data")):\n input_ids = data['input_ids'].to(device, dtype=torch.long)\n attention_mask = data['attention_mask'].to(device, dtype=torch.long)\n labels = data['labels'].to(device, dtype=torch.long)\n outputs, dec_self_attns, loss = model(input_ids, attention_mask, labels)\n running_loss += loss.item()\n return running_loss \/ len(val_loader)\n\ndef main():\n train_json_path = "data\/train.json" # \u8bad\u7ec3\u96c6\n val_json_path = "data\/val.json" # \u9a8c\u8bc1\u96c6\n vocab_path = "data\/vocab.json" # \u8bcd\u8868\u4f4d\u7f6e\n max_length = 120 # \u6700\u5927\u957f\u5ea6\n epochs = 15 # \u8fed\u4ee3\u5468\u671f\n batch_size = 128 # \u8bad\u7ec3\u4e00\u4e2a\u6279\u6b21\u7684\u5927\u5c0f\n lr = 1e-4 # \u5b66\u4e60\u7387\n model_output_dir = "output" # \u6a21\u578b\u4fdd\u5b58\u76ee\u5f55\n logs_dir = "logs" # \u65e5\u5fd7\u8bb0\u5f55\u76ee\u6807\n # \u8bbe\u5907\n device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")\n # \u52a0\u8f7d\u5206\u8bcd\u5668\n tokenizer = Tokenizer(vocab_path)\n # \u6a21\u578b\u53c2\u6570\n model_param = {\n "d_model": 768, # \u5d4c\u5165\u5c42\u5927\u5c0f\n "d_ff": 2048, # \u4e13\u5bb6\u7f51\u7edc\u7684\u5927\u5c0f\n "d_k": 64, # K \u7684\u5927\u5c0f\n "d_v": 64, # V \u7684\u5927\u5c0f\n "n_layers": 6, # \u89e3\u7801\u5c42\u7684\u6570\u91cf\n "n_heads": 8, # \u591a\u5934\u6ce8\u610f\u529b\u7684\u5934\u6570\n "max_pos": 1800, # \u4f4d\u7f6e\u7f16\u7801\u7684\u957f\u5ea6\n "device": device, # \u8bbe\u5907\n "vocab_size": tokenizer.get_vocab_size(), # \u8bcd\u8868\u5927\u5c0f\n "num_experts" :8, # 8\u4e2a\u4e13\u5bb6\n "top_k" : 2, # \u6bcf\u4e2atoken\u9009\u62e92\u4e2a\u4e13\u5bb6\n "load_balancing_weight" : 0.01 # \u8d1f\u8f7d\u5747\u8861\u635f\u5931\u6743\u91cd\n }\n model = GPTMoEModel(**model_param)\n print("Start Load Train Data...")\n train_params = {\n "batch_size": batch_size,\n "shuffle": True,\n "num_workers": 4,\n }\n training_set = QADataset(train_json_path, tokenizer, max_length)\n training_loader = DataLoader(training_set, **train_params)\n print("Start Load Validation Data...")\n val_params = {\n "batch_size": batch_size,\n "shuffle": False,\n "num_workers": 4,\n }\n val_set = QADataset(val_json_path, tokenizer, max_length)\n val_loader = DataLoader(val_set, **val_params)\n # \u65e5\u5fd7\u8bb0\u5f55\n writer = SummaryWriter(logs_dir)\n # \u4f18\u5316\u5668\n optimizer = torch.optim.AdamW(params=model.parameters(), lr=lr)\n model = model.to(device)\n # \u5f00\u59cb\u8bad\u7ec3\n print("Start Training...")\n train_model(\n model=model,\n train_loader=training_loader,\n val_loader=val_loader,\n optimizer=optimizer,\n device=device,\n num_epochs=epochs,\n model_output_dir=model_output_dir,\n writer=writer\n )\n writer.close()\n\nif __name__ == '__main__':\n main()<\/code><\/pre>\n\u8bad\u7ec3\u8fc7\u7a0b\uff1a<\/p>\n
![\"\u56fe\u7247\"](\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251107135250103.png\")
<\/p>\n
\u8bad\u7ec3\u7ed3\u679c\u540e\u4f7f\u7528 tensorboard<\/code> \u67e5\u770b\u4e0b loss<\/code> \u8d8b\u52bf\uff1a<\/p>\n![\"\u56fe\u7247\"](\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251107135250219.png\")
<\/p>\n
\u5728\u8bad\u7ec3 15<\/code>\u4e2aepochs<\/code> \u60c5\u51b5\u4e0b\uff0c\u9a8c\u8bc1\u96c6\u7684 loss<\/code>\uff0c\u5728\u524d 9<\/code> \u4e2a epochs<\/code>\u4e00\u76f4\u5904\u4e8e\u4e0b\u964d\u8d8b\u52bf\uff0c\u7b2c10<\/code>\u4e2aepochs<\/code>\u5f00\u59cb\u4e0a\u5347\uff0c\u8003\u8651\u51fa\u73b0\u8fc7\u62df\u5408\u60c5\u51b5\uff0c\u540e\u7eed\u4f18\u5316\u53ef\u4ee5\u5728\u7f51\u7edc\u4e2d\u52a0\u5165\u90e8\u5206 dropout<\/code> \u6765\u968f\u673a\u5931\u6d3b\u3002<\/p>\n\u56db\u3001\u6a21\u578b\u9884\u6d4b\u4f7f\u7528\u6d4b\u8bd5<\/h2>\nimport torch\n\nfrom model_moe import GPTMoEModel\nfrom tokenizer import Tokenizer\n\ndef generate(model, tokenizer, text, max_length, device):\n input, att_mask = tokenizer.encode(text)\n input = torch.tensor(input, dtype=torch.long, device=device).unsqueeze(0)\n stop = False\n input_len = len(input[0])\n while not stransform: translateY(\n if len(input[0]) - input_len > max_length:\n next_symbol = tokenizer.sep_token\n input = torch.cat(\n [input.detach(), torch.tensor([[next_symbol]], dtype=input.dtype, device=device)], -1)\n break\n projected, self_attns = model(input)\n prob = projected.squeeze(0).max(dim=-1, keepdim=False)[1]\n next_word = prob.data[-1]\n next_symbol = next_word\n if next_symbol == tokenizer.sep_token:\n stop = True\n input = torch.cat(\n [input.detach(), torch.tensor([[next_symbol]], dtype=input.dtype, device=device)], -1)\n decode = tokenizer.decode(input[0].tolist())\n decode = decode[len(text):]\n return "".join(decode)\n\ndef main():\n model_path = "output\/last.pt"\n vocab_path = "data\/vocab.json" # \u8bcd\u8868\u4f4d\u7f6e\n max_length = 120 # \u6700\u5927\u957f\u5ea6\n device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")\n # \u52a0\u8f7d\u5206\u8bcd\u5668\n tokenizer = Tokenizer(vocab_path)\n # \u6a21\u578b\u53c2\u6570\n model_param = {\n "d_model": 768, # \u5d4c\u5165\u5c42\u5927\u5c0f\n "d_ff": 2048, # \u4e13\u5bb6\u7f51\u7edc\u7684\u5927\u5c0f\n "d_k": 64, # K \u7684\u5927\u5c0f\n "d_v": 64, # V \u7684\u5927\u5c0f\n "n_layers": 6, # \u89e3\u7801\u5c42\u7684\u6570\u91cf\n "n_heads": 8, # \u591a\u5934\u6ce8\u610f\u529b\u7684\u5934\u6570\n "max_pos": 1800, # \u4f4d\u7f6e\u7f16\u7801\u7684\u957f\u5ea6\n "device": device, # \u8bbe\u5907\n "vocab_size": tokenizer.get_vocab_size(), # \u8bcd\u8868\u5927\u5c0f\n "num_experts": 8, # 8\u4e2a\u4e13\u5bb6\n "top_k": 2, # \u6bcf\u4e2atoken\u9009\u62e92\u4e2a\u4e13\u5bb6\n "load_balancing_weight": 0.01 # \u8d1f\u8f7d\u5747\u8861\u635f\u5931\u6743\u91cd\n }\n model = GPTMoEModel(**model_param)\n model.load_state_dict(torch.load(model_path))\n model.to(device)\n\n while True:\n text = input("\u8bf7\u8f93\u5165\uff1a")\n if not text:\n continue\n if text == "q":\n break\n res = generate(model, tokenizer, text, max_length, device)\n print("AI: ", res)\n\nif __name__ == '__main__':\n main()<\/code><\/pre>\n\u9884\u6d4b\u6548\u679c\uff1a<\/p>\n
![\"\u56fe\u7247\"](\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251107135250380.png\")
<\/p>\n
\u4e94\u3001\u603b\u7ed3<\/h2>\n
\u6587\u672c\u4ec5\u5bf9MOE<\/code>\u67b6\u6784\u505a\u4e86\u4e0b\u7684\u5b9e\u9a8c\uff0c\u5176\u4e2d\u8fd8\u6709\u5f88\u591a\u53ef\u4ee5\u4f18\u5316\u7684\u5730\u65b9\uff0c\u4f8b\u5982\u53ef\u4ee5\u4f7f\u7528RoPE<\/code>\u65cb\u8f6c\u4f4d\u7f6e\u7f16\u7801\u3001\u52a0\u5165 RMSNormal<\/code> \u3001\u5c1d\u8bd5\u66f4\u5148\u8fdb\u7684\u8def\u7531\u7b56\u7565\u3001\u52a0\u5165 dropout<\/code> \u7b49\u7b49\uff0c\u540e\u7eed\u4f60\u53ef\u4ee5\u7ee7\u7eed\u5c1d\u8bd5\u8fdb\u884c\u6539\u9020\u548c\u4f18\u5316\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"\u4e00\u3001\u57fa\u4e8e PyTorch \u4ece\u96f6\u624b\u6413 GPT \u6df7\u5408\u4e13\u5bb6 (MOE) \u5bf9\u8bdd\u6a21\u578b \u6df7\u5408\u4e13\u5bb6\u6a21\u578b\uff08MOE\uff09\u662f\u4e00\u79cd Transformer \u795e\u7ecf\u7f51 \u2026 …<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"ngg_post_thumbnail":0},"categories":[225],"tags":[],"_links":{"self":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9463"}],"collection":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=9463"}],"version-history":[{"count":1,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9463\/revisions"}],"predecessor-version":[{"id":9464,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9463\/revisions\/9464"}],"wp:attachment":[{"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=9463"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=9463"},{"taxonomy":"post_tag","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=9463"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}
![\"\u56fe\u7247\"](\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640.png\")
<\/p>\n