{"id":9506,"date":"2025-11-12T14:56:34","date_gmt":"2025-11-12T06:56:34","guid":{"rendered":"\/?p=9506"},"modified":"2025-11-12T14:56:34","modified_gmt":"2025-11-12T06:56:34","slug":"%e4%b8%ba%e4%bb%80%e4%b9%88%e8%af%b4%e5%a4%9a%e6%a8%a1%e6%80%81%e6%98%af%e6%8e%a8%e8%8d%90%e7%b3%bb%e7%bb%9f%e7%a0%b4%e5%b1%80%e7%9a%84%e5%85%b3%e9%94%ae%ef%bc%9f%e6%9d%a5%e8%87%aa%e9%a5%bf%e4%ba%86","status":"publish","type":"post","link":"\/?p=9506","title":{"rendered":"\u4e3a\u4ec0\u4e48\u8bf4\u591a\u6a21\u6001\u662f\u63a8\u8350\u7cfb\u7edf\u7834\u5c40\u7684\u5173\u952e\uff1f\u6765\u81ea\u997f\u4e86\u4e48\u4e00\u7ebf\u7684\u5b9e\u6218\u590d\u76d8"},"content":{"rendered":"<h2>\u5f15\u8a00<\/h2>\n<p>\u63a8\u8350\u7cfb\u7edf\u4f5c\u4e3a\u4e92\u8054\u7f51\u65f6\u4ee3\u8fde\u63a5\u7528\u6237\u4e0e\u4fe1\u606f\u7684\u6838\u5fc3\u6280\u672f\uff0c\u6b63\u5728\u7ecf\u5386\u4ece\u4f20\u7edf\u534f\u540c\u8fc7\u6ee4\u5411\u591a\u6a21\u6001\u667a\u80fd\u63a8\u8350\u7684\u91cd\u8981\u53d8\u9769\u3002\u968f\u7740\u6df1\u5ea6\u5b66\u4e60\u6280\u672f\u7684\u5feb\u901f\u53d1\u5c55\uff0c\u7279\u522b\u662f\u5927\u8bed\u8a00\u6a21\u578b\u548c\u591a\u6a21\u6001\u9884\u8bad\u7ec3\u6280\u672f\u7684\u6210\u719f\uff0c\u63a8\u8350\u7cfb\u7edf\u5f00\u59cb\u4ece\u5355\u7eaf\u4f9d\u8d56\u7528\u6237\u884c\u4e3aID\u7279\u5f81\u8f6c\u5411\u5145\u5206\u5229\u7528\u5546\u54c1\u56fe\u50cf\u3001\u6587\u672c\u63cf\u8ff0\u7b49\u4e30\u5bcc\u5185\u5bb9\u4fe1\u606f\u7684\u65b0\u8303\u5f0f\u3002<\/p>\n<p>\u7b14\u8005\u6709\u5e78\u4e8e24\u5e74\u4e0b\u534a\u5e74\u548c25\u5e74\u521d\u6df1\u5ea6\u53c2\u4e0e\u4e86\u997f\u4e86\u4e48\u9996\u9875\u5e97\u94fa\u63a8\u8350\u573a\u666f\u7684\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u8bbe\u8ba1\u4e0e\u5b9e\u73b0\uff0c\u4ee5\u53ca\u751f\u6210\u5f0franking\u524d\u671f\u6280\u672f\u65b9\u6848\u7684\u63a2\u7d22\u5b9e\u8df5\u3002\u5728\u6b64\u8fc7\u7a0b\u4e2d\uff0c\u5728\u591a\u6a21\u6001\u63a8\u8350\u6280\u672f\u53d1\u5c55\u8109\u7edc\u3001\u5b9e\u8df5\u6311\u6218\u4ee5\u53ca\u751f\u6210\u5f0f\u63a8\u8350\u6280\u672f\u65b9\u6cd5\u63a2\u7d22\u7b49\u65b9\u9762\u6536\u83b7\u9887\u4e30\u3002\u8d81\u774025\u5e74\u4e0a\u534a\u5e74\u603b\u7ed3\u7684\u673a\u4f1a\uff0c\u5e0c\u671b\u5c06\u8fd9\u4e9b\u5b66\u4e60\u8def\u5f84\u3001\u6280\u672f\u63a2\u7d22\u8fc7\u7a0b\u3001\u601d\u8003\u611f\u609f\u4ee5\u53ca\u5b9e\u8df5\u7ecf\u9a8c\u8fdb\u884c\u7cfb\u7edf\u6027\u7684\u5b8c\u6574\u56de\u987e\uff0c\u6574\u7406\u6210\u6587\u4f5c\u4e3a\u5b66\u4e60\u7b14\u8bb0\u4e0e\u5927\u5bb6\u5206\u4eab\u4ea4\u6d41\u3002<\/p>\n<p>\u6587\u7ae0\u4e3b\u8981\u5185\u5bb9\u5305\u62ec\uff1a<\/p>\n<ul>\n<li>\u6280\u672f\u6f14\u8fdb\u68b3\u7406\uff1a\u4ece\u4f20\u7edf\u63a8\u8350\u7b97\u6cd5\u5411\u591a\u6a21\u6001\u63a8\u8350\u7684\u53d1\u5c55\u5386\u7a0b\uff0c\u4ee5\u53ca\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u7684\u6838\u5fc3\u65b9\u6cd5\uff1b<\/li>\n<li>\u5b9e\u8df5\u7ecf\u9a8c\u603b\u7ed3\uff1a\u57fa\u4e8e\u997f\u4e86\u4e48\u573a\u666f\u7684\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u5b8c\u6574\u8bbe\u8ba1\u4e0e\u5b9e\u73b0\u8fc7\u7a0b\uff1b<\/li>\n<li>\u524d\u6cbf\u63a2\u7d22\u5206\u4eab\uff1a\u751f\u6210\u5f0f\u63a8\u8350\u7684\u6280\u672f\u5c1d\u8bd5\u4e0e\u4e1a\u754c\u53d1\u5c55\u8d8b\u52bf\uff1b<\/li>\n<\/ul>\n<p>\u5e0c\u671b\u8fd9\u4efd\u603b\u7ed3\u80fd\u591f\u4e3a\u540c\u6837\u5728\u591a\u6a21\u6001\/\u751f\u6210\u5f0f\u63a8\u8350\u9886\u57df\u63a2\u7d22\u7684\u6280\u672f\u540c\u5b66\u63d0\u4f9b\u4e00\u4e9b\u53c2\u8003\u3002<\/p>\n<h2>\u4e00\u3001\u4f20\u7edf\u63a8\u8350\u7b97\u6cd5\u5411\u591a\u6a21\u6001\u7684\u6f14\u8fdb<\/h2>\n<h3><strong>1.1 \u4f20\u7edf\u65b9\u6cd5\u56de\u987e<\/strong><\/h3>\n<p>\u63a8\u8350\u7cfb\u7edf\u7684\u57fa\u7840\u67b6\u6784\u901a\u5e38\u7531\u7528\u6237\u7279\u5f81\u3001\u7269\u6599\u7279\u5f81\u548c\u4ea4\u4e92\u9884\u6d4b\u6a21\u5757\u7ec4\u6210\u3002\u5728\u4f20\u7edf\u57fa\u4e8e\u6df1\u5ea6\u795e\u7ecf\u7f51\u7edc\uff08DNN\uff09\u7684\u63a8\u8350\u7cfb\u7edf\u4e2d\uff0c\u7279\u5f81\u5de5\u7a0b\u662f\u6838\u5fc3\u73af\u8282\uff0c\u5176\u4e2dID\u7c7b\u7279\u5f81\u7684\u5904\u7406\u5c24\u4e3a\u91cd\u8981\uff0c\u5176\u5e38\u89c1\u7c7b\u522b\u4e0e\u5904\u7406\u6d41\u7a0b\u5982\u4e0b\uff1a<\/p>\n<p><strong>\u7528\u6237ID\u7279\u5f81\uff1a<\/strong><\/p>\n<ul>\n<li>\u57fa\u672c\u5c5e\u6027\uff1a \u5e74\u9f84\uff08\u5206\u6876\u6570\u503c\u5316\uff09\u3001\u5730\u57df\u7b49\u6807\u7b7e\u5316\u7279\u5f81\uff1b<\/li>\n<li>\u5386\u53f2\u884c\u4e3a\u5e8f\u5217\uff1a \u7528\u6237\u5386\u53f2\u4ea4\u4e92\u7684item\u5e8f\u5217\uff0c\u901a\u8fc7Embedding\u5c42\u548c\u6c60\u5316\u64cd\u4f5c\u5f62\u6210\u7efc\u5408\u8868\u5f81\uff1b<\/li>\n<\/ul>\n<p><strong>\u7269\u6599ID\u7279\u5f81\uff1a<\/strong><\/p>\n<ul>\n<li>\u52a8\u6001\u5c5e\u6027\uff1a \u9500\u91cf\u3001\u8bc4\u5206\u3001\u536b\u751f\u7b49\u7ea7\u7b49\u5b9e\u65f6\u4e1a\u52a1\u6307\u6807\uff1b<\/li>\n<li>\u9759\u6001\u5c5e\u6027\uff1a \u7c7b\u76ee\u3001\u54c1\u724c\u7b49\u56fa\u6709\u7279\u5f81\uff1b<\/li>\n<\/ul>\n<p><strong>\u7279\u5f81\u7f16\u7801\uff1a<\/strong><\/p>\n<ul>\n<li>\u7f16\u7801\u6d41\u7a0b\uff1a\u539f\u59cb\u7279\u5f81 \u2192 Hash\u7f16\u7801 \u2192 Embedding\u5c42 \u2192 \u7279\u5f81\u5411\u91cf\uff1b<\/li>\n<li>\u5386\u53f2\u5e8f\u5217\u5904\u7406\uff1a\u5386\u53f2\u884c\u4e3a\u5e8f\u5217\u4e2d\u7684item\u5206\u522b\u8fdb\u884cEmbedding\u7f16\u7801\uff0c\u62fc\u63a5\u540e\u901a\u8fc7\u6c60\u5316\u751f\u6210\u5e8f\u5217\u6574\u4f53\u8868\u5f81\uff1b<\/li>\n<\/ul>\n<p>ID\u7279\u5f81\u7684Embedding\u5c42\u901a\u5e38\u4e0d\u8fdb\u884c\u72ec\u7acb\u8bad\u7ec3\uff0c\u800c\u662f\u4e0e\u4e0b\u6e38\u9884\u6d4b\u4efb\u52a1\u7aef\u5230\u7aef\u8054\u5408\u4f18\u5316\u3002\u4ee5\u4e0b\u662f\u4e00\u4e2a\u5178\u578b\u7684\u8bad\u7ec3\u67b6\u6784\u793a\u4f8b\uff1a<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143517617.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>source: <a href=\"https:\/\/zhuanlan.zhihu.com\/p\/570911435\">https:\/\/zhuanlan.zhihu.com\/p\/570911435<\/a><\/p>\n<p><strong>\u7279\u5f81\u5904\u7406\u8bf4\u660e\uff1a<\/strong><\/p>\n<ol>\n<li>\u7528\u6237\u57fa\u7840\u7279\u5f81\uff1a\u76f4\u63a5\u8f93\u5165Embedding\u5c42\uff1b<\/li>\n<li>\u7528\u6237\u5386\u53f2\u884c\u4e3a\u7279\u5f81\uff1a\u5e8f\u5217item\u5206\u522bEmbedding\u540e\u62fc\u63a5\uff0c\u7ecf\u6c60\u5316\u5f62\u6210\u7efc\u5408\u8868\u5f81\uff1b<\/li>\n<li>\u5019\u9009item\u7279\u5f81\uff1a\u5f53\u524d\u66dd\u5149\u7269\u6599\u7684\u7279\u5f81\u5411\u91cf\uff1b<\/li>\n<\/ol>\n<p><strong>\u8bad\u7ec3\u4efb\u52a1\u8bbe\u8ba1\uff1a<\/strong><\/p>\n<ul>\n<li>CTR\u4efb\u52a1\uff1a\u9884\u6d4b\u7528\u6237\u5bf9\u6bcf\u4e2a\u66dd\u5149item\u7684\u70b9\u51fb\u6982\u7387\uff1b<\/li>\n<li>CVR\u4efb\u52a1\uff1a\u9884\u6d4b\u7528\u6237\u5bf9\u6bcf\u4e2a\u66dd\u5149item\u4ea7\u751f\u8f6c\u5316\u884c\u4e3a\u7684\u6982\u7387\uff08\u5982\u4e0b\u5355\/\u6ce8\u518c\uff09\uff1b<\/li>\n<\/ul>\n<p><strong>\u5b9e\u73b0\u65b9\u5f0f\uff1a<\/strong><\/p>\n<ul>\n<li>Embedding\u5c42\u5728CTR\/CVR\u4efb\u52a1\u95f4\u5171\u4eab\uff1b<\/li>\n<li>\u9884\u6d4b\u5c42\u5206\u8bbe\u72ec\u7acbMLP\u7f51\u7edc\u9002\u5e94\u4e0d\u540c\u4efb\u52a1\u7279\u6027\uff1b<\/li>\n<\/ul>\n<p><strong>\u8bc4\u4f30\u6307\u6807\uff1a<\/strong><\/p>\n<ul>\n<li>AUC\uff08Area Under Curve\uff09\uff1a\u8861\u91cf\u6a21\u578b\u533a\u5206\u6b63\u8d1f\u6837\u672c\u7684\u80fd\u529b\uff1b<\/li>\n<li>\u79bb\u7ebf\u6307\u6807\u4e0e\u5728\u7ebf\u6548\u679c\u7684\u5173\u8054\u6027\u9a8c\u8bc1\uff1b<\/li>\n<li>A\/B\u6d4b\u8bd5\u7b49\u5728\u7ebf\u8bc4\u4f30\u65b9\u6cd5\uff1b<\/li>\n<\/ul>\n<p>\u7136\u800c\uff0c\u4f20\u7edf\u67b6\u6784\u7684ID\u7279\u5f81\u65b9\u6cd5\u5b58\u5728\u4e00\u4e9b\u672c\u8d28\u5c40\u9650\uff1a\u5982\u65b0\u7269\u6599\u7f3a\u4e4f\u4ea4\u4e92\u5386\u53f2\u5bfc\u81f4\u7684\u51b7\u542f\u52a8\u95ee\u9898\uff0c\u4ee5\u53caID\u7279\u5f81\u65e0\u6cd5\u6355\u6349\u7269\u6599\u5185\u5bb9\u8bed\u4e49\u4fe1\u606f\u7684\u95ee\u9898\u3002\u8fd9\u4e9b\u5c40\u9650\u6027\u63a8\u52a8\u4e86\u63a8\u8350\u7b97\u6cd5\u7684\u6301\u7eed\u6f14\u8fdb\u548c\u521b\u65b0\u3002<\/p>\n<h3><strong>1.2 \u6280\u672f\u6f14\u8fdb<\/strong><\/h3>\n<h3>1.2.1 \u63a8\u8350\u7b97\u6cd5\u7684\u591a\u5143\u5316\u53d1\u5c55\uff082018-2023\uff09<\/h3>\n<p><strong>\u6ce8\u610f\u529b\u673a\u5236\u5f15\u5165\uff1a<\/strong>DIN\uff08KDD 2018\uff09[1]\u5728\u4f20\u7edfEmbedding+MLP\u8303\u5f0f\u57fa\u7840\u4e0a\uff0c\u901a\u8fc7\u5f15\u5165\u76ee\u6807\u6ce8\u610f\u529b\u673a\u5236\uff08Target Attention\uff09\u5b9e\u73b0\u7528\u6237\u5174\u8da3\u7684\u52a8\u6001\u5efa\u6a21\u3002\u6bcf\u4e2a\u5386\u53f2item\u4e0e\u5019\u9009item\u8fdb\u884c\u5411\u91cf\u51cf\u6cd5\u8fd0\u7b97\u540e\u4e0e\u539f\u59cb\u5e8f\u5217\u5411\u91cf\u62fc\u63a5\uff0c\u8f93\u5165\u5168\u8fde\u63a5\u7f51\u7edc\uff0c\u751f\u6210\u53cd\u6620\u7528\u6237\u52a8\u6001\u5174\u8da3\u7684\u6700\u7ec8\u8868\u5f81\uff0c\u7a81\u7834\u4e86\u7b80\u5355\u6c60\u5316\u7684\u4fe1\u606f\u635f\u5931\u9650\u5236\u3002<\/p>\n<p><strong>\u56fe\u795e\u7ecf\u7f51\u7edc\u5e94\u7528\uff1a<\/strong>\u4ee5LightGCN\uff08SIGIR 2020\uff09[2]\u4e3a\u4ee3\u8868\uff0c\u901a\u8fc7\u5efa\u6a21\u7528\u6237-\u7269\u54c1\u4ea4\u4e92\u56fe\u6355\u6349\u9ad8\u9636\u534f\u540c\u8fc7\u6ee4\u4fe1\u53f7\uff0c\u7b80\u5316\u56fe\u5377\u79ef\u7f51\u7edc\u7ed3\u6784\u5e76\u663e\u8457\u63d0\u5347\u63a8\u8350\u6548\u679c\u3002<\/p>\n<p><strong>\u5bf9\u6bd4\u5b66\u4e60\u5f15\u5165\uff1a<\/strong>\u53d7\u8ba1\u7b97\u673a\u89c6\u89c9\u542f\u53d1\uff0cSGL\uff08SIGIR 2021\uff09[3]\u548cCL4SRec\uff08ICDE 2022\uff09[4]\u7b49\u5de5\u4f5c\u5c06\u5bf9\u6bd4\u5b66\u4e60\u5e94\u7528\u4e8e\u63a8\u8350\u7cfb\u7edf\uff0c\u901a\u8fc7\u6570\u636e\u589e\u5f3a\u7b56\u7565\u7f13\u89e3\u6570\u636e\u7a00\u758f\u6027\u95ee\u9898\u3002<\/p>\n<p><strong>\u9884\u8bad\u7ec3\u8303\u5f0f\u63a2\u7d22\uff1a<\/strong>UniSRec\uff08CIKM 2022\uff09[5]\u548cP5\uff08RecSys 2022\uff09[6]\u7b49\u5de5\u4f5c\u501f\u9274NLP\u9884\u8bad\u7ec3\u7ecf\u9a8c\uff0c\u5b9e\u73b0\u8de8\u57df\u77e5\u8bc6\u8fc1\u79fb\u548c\u7edf\u4e00\u4efb\u52a1\u5efa\u6a21\uff0c\u4e3a\u63a8\u8350\u7cfb\u7edf\u5f15\u5165\u4e86\u9884\u8bad\u7ec3-\u5fae\u8c03\u7684\u65b0\u8303\u5f0f\u3002<\/p>\n<h3>1.2.2 \u5927\u8bed\u8a00\u6a21\u578b\u4e0e\u591a\u6a21\u6001\u63a8\u8350\u7684\u7ed3\u5408\u63a2\u7d22\uff082023-2024\uff09<\/h3>\n<p>\u4e0a\u8ff0\u6280\u672f\u6f14\u8fdb\u867d\u7136\u5728\u6a21\u578b\u67b6\u6784\u548c\u5b66\u4e60\u8303\u5f0f\u4e0a\u53d6\u5f97\u4e86\u663e\u8457\u8fdb\u5c55\uff0c\u4f46\u672c\u8d28\u4e0a\u4ecd\u5c40\u9650\u4e8eID\u7279\u5f81\u548c\u5c11\u91cf\u7ed3\u6784\u5316\u7279\u5f81\u7684\u5efa\u6a21\u3002\u968f\u7740\u6df1\u5ea6\u5b66\u4e60\u5728\u8ba1\u7b97\u673a\u89c6\u89c9\u548c\u81ea\u7136\u8bed\u8a00\u5904\u7406\u9886\u57df\u7684\u7a81\u7834\uff0c\u7279\u522b\u662f\u9884\u8bad\u7ec3\u5927\u6a21\u578b\u7684\u6210\u719f\uff0c\u63a8\u8350\u7cfb\u7edf\u5f00\u59cb\u63a2\u7d22\u5982\u4f55\u5229\u7528\u5546\u54c1\u56fe\u7247\u3001\u63cf\u8ff0\u6587\u672c\u7b49\u4e30\u5bcc\u7684\u591a\u6a21\u6001\u5185\u5bb9\u4fe1\u606f\u3002<\/p>\n<p>2023\u5e74\u4ee5\u6765\uff0c\u5927\u8bed\u8a00\u6a21\u578b\u6280\u672f\u7684\u5feb\u901f\u53d1\u5c55\u4e3a\u591a\u6a21\u6001\u63a8\u8350\u63d0\u4f9b\u4e86\u65b0\u7684\u6280\u672f\u57fa\u7840\uff0c\u63a8\u8350\u7b97\u6cd5\u5f00\u59cb\u4ece\u7eafID\u7279\u5f81\u5efa\u6a21\u8f6c\u5411ID\u7279\u5f81\u4e0e\u591a\u6a21\u6001\u5185\u5bb9\u7279\u5f81\u7684\u6df1\u5ea6\u878d\u5408\u3002<\/p>\n<h4><strong>1.2.2.1 \u591a\u6a21\u6001\u63a8\u8350\u7684\u4ef7\u503c\u4e0e\u6311\u6218<\/strong><\/h4>\n<p>\u591a\u6a21\u6001\u7279\u5f81\u7684\u5f15\u5165\u4e3b\u8981\u89e3\u51b3\u4e86\u4f20\u7edf\u63a8\u8350\u7cfb\u7edf\u7684\u4e09\u5927\u75db\u70b9\uff1a<strong>\u957f\u5c3eitem\u548c\u957f\u5c3e\u7528\u6237\u7684\u884c\u4e3a\u5e8f\u5217\u6570\u636e\u4e0d\u8db3\u95ee\u9898**<\/strong>\u3001<strong><strong>\u65b0item\u548c\u65b0\u7528\u6237\u7684\u51b7\u542f\u52a8\u95ee\u9898<\/strong><\/strong>\u3001<strong><strong>\u8de8\u573a\u666f\u7684\u63a8\u8350\u7279\u5f81\u8fc1\u79fb\u95ee\u9898<\/strong><\/strong>\u3002**<\/p>\n<p>\u7136\u800c\uff0c\u591a\u6a21\u6001\u63a8\u8350\u9762\u4e34\u4e24\u4e2a\u6838\u5fc3\u6280\u672f\u6311\u6218\uff1a\u4e00\u662f<strong>\u5982\u4f55\u4f7f\u591a\u6a21\u6001\u8868\u5f81\u5bf9\u9f50\u63a8\u8350\u4e2d\u7684user-item\u4ea4\u4e92\u5173\u7cfb<\/strong>\uff0c\u5373\u5728\u8bed\u4e49\u5c42\u9762\u5b9a\u4e49\u63a8\u8350\u4e2d\u7684\u76f8\u4f3citem\uff1b\u4e8c\u662f<strong>\u7a20\u5bc6\u8bed\u4e49\u7279\u5f81\u4e0e\u7a00\u758fID\u7279\u5f81\u5982\u4f55\u8054\u5408\u8bad\u7ec3<\/strong>\uff0c\u4e24\u7c7b\u7279\u5f81\u5728\u7ef4\u5ea6\u3001\u66f4\u65b0\u9891\u7387\u3001\u8bad\u7ec3\u65b9\u5f0f\u7b49\u65b9\u9762\u5b58\u5728\u663e\u8457\u5dee\u5f02\u3002<\/p>\n<h4><strong>1.2.2.2 \u4ee3\u8868\u6027\u5de5\u4f5c\u53ca\u5176\u6280\u672f\u521b\u65b0<\/strong><\/h4>\n<p><strong>\u5c0f\u7ea2\u4e66AlignRec\uff082024.03\uff09**<\/strong>\uff1a**<\/p>\n<p>AlignRec: Aligning and Training in Multimodal Recommendations<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143518231.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>LLM2Vec<\/p>\n<p>AlignRec\u901a\u8fc7\u8054\u5408\u8bad\u7ec3\u7b56\u7565\u7cfb\u7edf\u6027\u5730\u89e3\u51b3\u4e86\u591a\u6a21\u6001\u63a8\u8350\u7684\u6838\u5fc3\u6311\u6218\uff1a<\/p>\n<ul>\n<li><strong>\u591a\u6a21\u6001\u7f16\u7801\u5668**<\/strong>\uff1a**\u4f7f\u7528BEIT-3\u4ea4\u53c9\u6ce8\u610f\u529b\u673a\u5236\u5b9e\u73b0\u89c6\u89c9\u4e0e\u6587\u672c\u7684\u7edf\u4e00\u8868\u793a\uff0c\u901a\u8fc7mask loss\u8fdb\u884c\u5bf9\u9f50\u8bad\u7ec3\uff1b<\/li>\n<li><strong>\u805a\u5408\u6a21\u5757**<\/strong>\uff1a**\u91c7\u7528LightGCN\u8fdb\u884c\u56fe\u7ed3\u6784\u5efa\u6a21\u3002\u8bbe\u8ba1\u7279\u5f81\u805a\u5408\u6a21\u5757\uff0c\u591a\u6a21\u6001\u7279\u5f81\u8fc7MLP+sigmoid\u964d\u7ef4\u540e\u4e0eID\u7279\u5f81\u505a\u5143\u7d20\u4e58\u6cd5\u5b9e\u73b0\u7b80\u5316\uff0c\u57fa\u4e8etopk\u76f8\u4f3citem\u8ba1\u7b97\u5305\u542bitem\u76f8\u4f3c\u5173\u7cfb\u7684\u591a\u6a21\u6001\u8868\u5f81\uff1b\u53d6\u7528\u6237\u70b9\u51fb\u5e8f\u5217item\u7279\u5f81\u548c\u4f5c\u4e3a\u7528\u6237\u591a\u6a21\u6001\u8868\u5f81\uff1b<\/li>\n<li><strong>\u878d\u5408\u6a21\u5757<\/strong>\uff1aID\u7279\u5f81\u4e0e\u5904\u7406\u540e\u7684\u591a\u6a21\u6001\u7279\u5f81\u878d\u5408\uff08\u5411\u91cf\u51cf\u6cd5\uff09\uff0c\u5f97\u5230\u7efc\u5408\u7684user\u7279\u5f81\u4e0eitem\u7279\u5f81\uff0c\u901a\u8fc7\u5bf9\u6bd4\u5b66\u4e60\u4f18\u5316\uff1b<\/li>\n<\/ul>\n<p><strong>\u9488\u5bf9\u6838\u5fc3\u6311\u6218\u7684\u89e3\u51b3\u601d\u8def**<\/strong>\uff1a**AlignRec\u91c7\u7528\u8054\u5408\u8bad\u7ec3\u5bf9\u9f50\u7b56\u7565\uff0c\u901a\u8fc7\u7279\u5f81\u805a\u5408\u6a21\u5757\u5c06ID\u7279\u5f81\u4e0e\u591a\u6a21\u6001\u7279\u5f81\u878d\u5408\u540e\uff0c\u57fa\u4e8eitem\u878d\u5408\u7279\u5f81\u76f8\u4f3c\u5ea6\u5173\u7cfb\u5efa\u6a21item\u8bed\u4e49\u7279\u5f81\uff0c\u5b9e\u73b0\u4e86\u6df1\u5ea6\u878d\u5408\u800c\u975e\u7b80\u5355\u62fc\u63a5\u3002<\/p>\n<p>\u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0cAlignRec\u901a\u8fc7\u56fe\u6587\u5bf9\u81ea\u8eabmask\u8bad\u7ec3\u5f97\u5230\u878d\u5408\u8868\u5f81\u7684\u65b9\u5f0f\uff0c\u5927\u81f4\u7b49\u4ef7\u4e8e\u56fe\u6587\u8868\u5f81\u76f4\u63a5\u76f8\u52a0\u800c\u7f3a\u4e4f\u5bf9\u6bd4\u5b66\u4e60\u3002\u8fd9\u79cd\u8bbe\u8ba1\u5bfc\u81f4\u6a21\u578b\u5728\u57fa\u7840\u8868\u5f81\u9636\u6bb5\u7f3a\u4e4f\u5bf9item\u6574\u4f53\u8868\u5f81\u95f4\u76f8\u4f3c\u5ea6\u5173\u7cfb\u7684\u7406\u89e3\uff0c\u56e0\u6b64\u9700\u8981\u5b8c\u5168\u4f9d\u9760\u540e\u7eed\u7684Aggregation Module\u901a\u8fc7LightGCN\u7b49\u590d\u6742\u673a\u5236\u6765\u91cd\u65b0\u5efa\u7acb\u5185\u5bb9\u4e0eID\u8868\u5f81\u7684\u5bf9\u9f50\u5173\u7cfb\u3002<\/p>\n<p>\u6b64\u5916\uff0c\u5c0f\u7ea2\u4e66\u7684Notellm\u7cfb\u5217\u5de5\u4f5c[8,9]\u4e5f\u540c\u6837\u503c\u5f97\u5173\u6ce8\uff0c\u611f\u5174\u8da3\u7684\u8bfb\u8005\u53ef\u8fdb\u4e00\u6b65\u9605\u8bfb\u76f8\u5173\u8bba\u6587\u3002<\/p>\n<p><strong>\u963f\u91cc\u5988\u5988SimTier &amp; MAKE\uff082024.07\uff09**<\/strong>\uff1a**<\/p>\n<p>Enhancing Taobao Display Advertising with Multimodal Representations: Challenges, Approaches and Insights<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143518966.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>SimTier &amp; MAKE\u63d0\u51fa\u4e86\u57fa\u4e8e\u8bed\u4e49\u4ea4\u4e92\u5173\u7cfb\u7684\u5bf9\u9f50\u7b56\u7565\uff0c\u5f3a\u8c03\u591a\u6a21\u6001\u7279\u5f81\u7684\u7eaf\u7cb9\u6027\uff1a<\/p>\n<ul>\n<li><strong>\u9884\u8bad\u7ec3\u9636\u6bb5**<\/strong>\uff1a**\u4f7f\u7528\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u94fe\u4f5c\u4e3a\u6b63\u6837\u672c\u6548\u679c\u6700\u4f73\u3002\u5173\u952e\u6d1e\u5bdf\u662f\u5b9a\u4e49\u8bed\u4e49\u76f8\u4f3c\u7684\u5546\u54c1\u5bf9\u65f6\uff0c\u9700\u8981\u4fdd\u8bc1\u8fd9\u4e00\u76f8\u4f3c\u6027\u80fd\u591f\u5f52\u56e0\u4e8e\u56fe\u50cf\u3001\u6587\u672c\u7684\u591a\u6a21\u6001\u8bed\u4e49\u76f8\u4f3c\u6027\uff0c\u800c\u4e0d\u662f\u5176\u4ed6\u65e0\u5173\u56e0\u7d20\uff08\u4f7f\u7528\u5176\u4ed6\u65e0\u5173\u56e0\u7d20\u4f1a\u5bfc\u81f4\u8868\u5f81\u504f\u79bb\u9884\u671f\u9000\u5316\u4e3aID\u8868\u5f81\uff09\u3002\u8d1f\u6837\u672c\u4f7f\u7528MoCo memory bank\u6269\u91cf\u81f3196,800\uff0c\u91c7\u7528InfoNCE loss\uff1b<\/li>\n<li><strong>\u96c6\u6210\u9636\u6bb5**<\/strong>\uff1a**\n<ul>\n<li>SimTier\uff1a\u6bcf\u4e2aitem\u4e0e\u5176\u4ed6\u6240\u6709item\u8ba1\u7b97\u8bed\u4e49\u76f8\u4f3c\u5ea6\uff0c\u5c06\u76f8\u4f3c\u5ea6\u7ed3\u679c\u5728[-1,1]\u5206\u4e3aN\u4e2a\u533a\u95f4\uff0c\u7edf\u8ba1\u6bcf\u4e2a\u533a\u95f4\u7684\u76f8\u4f3c\u5ea6\u7ed3\u679c\u6570\u91cf\uff0c\u5f97\u5230N\u7ef4\u5411\u91cf\u4f5c\u4e3a\u8f93\u51fa\uff0c\u7b80\u5316\u591a\u6a21\u6001\u8868\u5f81\u7684\u4f7f\u7528\u65b9\u5f0f\uff1b<\/li>\n<li>MAKE\uff1a\u9488\u5bf9ID\u6a21\u578b\u548c\u591a\u6a21\u6001\u6a21\u578b\u5728\u6cdb\u5316\u80fd\u529b\u4e0a\u7684\u5dee\u5f02\uff0c\u4f18\u5316\u4e24\u8005\u5728\u8bad\u7ec3epoch\u6570\u4e0a\u7684\u4e0d\u540c\u9700\u6c42\uff1b<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p><strong>\u9488\u5bf9\u6838\u5fc3\u6311\u6218\u7684\u89e3\u51b3\u601d\u8def**<\/strong>\uff1a**\u4e0eAlignRec\u4e0d\u540c\uff0cSimTier\u5b8c\u5168\u4ece\u8bed\u4e49\u5c42\u9762\u7740\u624b\u5b9a\u4e49\u76f8\u4f3citem\uff0c\u5229\u7528\u7528\u6237\u7684\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u94fe\u907f\u514d\u591a\u6a21\u6001\u7279\u5f81\u9000\u5316\u4e3aID\u7279\u5f81\u3002\u8fd9\u79cd\u7b56\u7565\u7684\u6838\u5fc3\u7406\u5ff5\u662f\u4fdd\u6301\u591a\u6a21\u6001\u7279\u5f81\u7684\u8bed\u4e49\u7eaf\u7cb9\u6027\uff0c\u907f\u514d\u88abID\u7279\u5f81\u7684\u534f\u540c\u8fc7\u6ee4\u4fe1\u53f7&quot;\u6c61\u67d3&quot;\u3002<\/p>\n<p><strong>\u5feb\u624bQARM\uff082024.11\uff09**<\/strong>\uff1a**<\/p>\n<p>QARM: Quantitative Alignment Multi-Modal Recommendation at Kuaishou<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143519587.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>QARM\u63d0\u51fa\u4e86\u57fa\u4e8eID\u4ea4\u4e92\u5173\u7cfb\u7684\u5bf9\u9f50\u7b56\u7565\uff0c\u5f3a\u8c03\u9002\u914d\u771f\u5b9e\u63a8\u8350\u573a\u666f\uff1a<\/p>\n<ul>\n<li><strong>Item\u5bf9\u9f50**<\/strong>\uff1a**\u5bf9\u591a\u6a21\u6001\u6a21\u578b\u4f7f\u7528\u4e0b\u6e38\u63a8\u8350\u7684\u4ea4\u4e92\u6570\u636e\u5fae\u8c03\uff0c\u4f7f\u591a\u6a21\u6001\u8868\u5f81\u66f4\u9002\u914d\u63a8\u8350\u573a\u666f\u771f\u5b9e\u7684User-Item\u4ea4\u4e92\u884c\u4e3a\u5206\u5e03\uff1b\n<ul>\n<li>\u57fa\u4e8eU2I\u53ec\u56de\u6a21\u578b\uff1a\u5bf9\u4e8e\u6bcf\u4e2a\u7528\u6237\u6b63\u5411\u70b9\u51fb\u7684\u76ee\u6807Item\uff0c\u4ece\u6700\u8fd1\u768450\u4e2a\u70b9\u51fbItem\u4e2d\u9009\u62e9\u4e0e\u76ee\u6807Item\u5728ID\u8868\u5f81\u7a7a\u95f4\u4e2d\u6700\u76f8\u4f3c\u7684Item\u4f5c\u4e3a\u89e6\u53d1Item\uff1b<\/li>\n<li>\u57fa\u4e8eI2I\u53ec\u56de\u6a21\u578b\uff1a\u5229\u7528\u73b0\u6709\u6a21\u578b\u5b66\u4e60\u5230\u7684\u5177\u6709\u9ad8\u76f8\u4f3c\u5ea6\u7684\u7a33\u5b9aItem Pair\u5bf9\u4f5c\u4e3a\u6570\u636e\u6e90\uff0c\u6bd4\u5982Swing\u53ec\u56de\uff1b<\/li>\n<\/ul>\n<\/li>\n<li><strong>\u91cf\u5316\u7f16\u7801**<\/strong>\uff1a**\u5c06\u5bf9\u9f50\u7684\u591a\u6a21\u6001\u8868\u5f81\u8f6c\u6362\u4e3a\u53ef\u8bad\u7ec3\u7684\u8bed\u4e49ID\u3002\u9488\u5bf9\u591a\u6a21\u6001\u8868\u5f81\u66f4\u65b0\u95ee\u9898\uff0c\u4e1a\u754c\u4e00\u79cd\u6700\u76f4\u63a5\u4e14\u5e7f\u6cdb\u5e94\u7528\u7684trick, \u662f\u4f7f\u7528\u591a\u6a21\u6001\u8868\u5f81\u6765\u6784\u9020\u8bed\u4e49\u76f8\u4f3cItem\u7684ID list, \u7136\u540e\u4ee5ID List\u4f5c\u4e3a\u8be5item\u7684\u8bed\u4e49\u7279\u5f81\u8f93\u5165\u5230\u63a8\u8350\u6a21\u578b, \u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u95f4\u63a5\u89c4\u907f\u4e86\u591a\u6a21\u6001\u8868\u5f81\u4e0d\u66f4\u65b0\u7684\u95ee\u9898\u3002QARM\u5c06\u4e0a\u8ff0trick\u505a\u5230\u6781\u81f4\uff1a\u9996\u5148\u5c06\u591a\u6a21\u6001\u8868\u5f81\u91cf\u5316\u538b\u7f29\u6210\u8bed\u4e49ID\uff0c\u518d\u5c06\u8bed\u4e49ID\u8f93\u5165\u5230\u63a8\u8350\u6a21\u578b\u4e2d\u3002<\/li>\n<\/ul>\n<p><strong>\u9488\u5bf9\u6838\u5fc3\u6311\u6218\u7684\u89e3\u51b3\u601d\u8def**<\/strong>\uff1a**QARM\u91c7\u7528\u57fa\u4e8eID\u7279\u5f81\u4ea4\u4e92\u5173\u7cfb\u5b9a\u4e49\u76f8\u4f3citem\u7684\u7b56\u7565\uff0c\u4f7f\u591a\u6a21\u6001\u8868\u5f81\u66f4\u9002\u914dUser-Item\u4ea4\u4e92\u884c\u4e3a\u5206\u5e03\u3002\u8fd9\u4e0e\u963f\u91cc\u5988\u5988\u7684\u65b9\u6848\u5f62\u6210\u4e86\u6709\u8da3\u7684\u5bf9\u6bd4\uff1aQARM\u8ba4\u4e3a\u5e94\u8be5\u8ba9\u591a\u6a21\u6001\u7279\u5f81\u9002\u914d\u73b0\u6709\u7684\u63a8\u8350\u4ea4\u4e92\u6a21\u5f0f\uff0c\u800c\u963f\u91cc\u5988\u5988\u8ba4\u4e3a\u8fd9\u6837\u4f1a\u4f7f\u591a\u6a21\u6001\u7279\u5f81\u9000\u5316\u4e3aID\u7279\u5f81\uff0c\u5e94\u8be5\u5b8c\u5168\u4ece\u8bed\u4e49\u5c42\u9762\u5b9a\u4e49\u76f8\u4f3c\u6027\u3002<\/p>\n<h4><strong>1.2.2.3 \u6280\u672f\u8def\u7ebf\u603b\u7ed3\u4e0e\u53d1\u5c55\u8d8b\u52bf<\/strong><\/h4>\n<p>\u4ece\u8fd9\u4e9b\u5de5\u4f5c\u53ef\u4ee5\u770b\u51fa\uff0c\u4e1a\u754c\u5728\u591a\u6a21\u6001\u63a8\u8350\u7684\u4e24\u5927\u6838\u5fc3\u6311\u6218\u4e0a\u5f62\u6210\u4e86\u4e0d\u540c\u7684\u6280\u672f\u8def\u7ebf\uff1a<\/p>\n<p><strong>\u591a\u6a21\u6001\u8868\u5f81\u5bf9\u9f50\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u57fa\u4e8eID\u4ea4\u4e92\u5173\u7cfb\u5bf9\u9f50\uff08\u5feb\u624bQARM\uff09\uff1a\u9002\u914d\u73b0\u6709\u63a8\u8350\u884c\u4e3a\u5206\u5e03\uff0c\u4f46\u53ef\u80fd\u5bfc\u81f4\u591a\u6a21\u6001\u7279\u5f81\u9000\u5316\uff1b<\/li>\n<li>\u57fa\u4e8e\u8bed\u4e49\u4ea4\u4e92\u5173\u7cfb\u5bf9\u9f50\uff08\u963f\u91cc\u5988\u5988\uff09\uff1a\u4fdd\u6301\u8bed\u4e49\u7eaf\u7cb9\u6027\uff0c\u4f46\u53ef\u80fd\u8131\u79bb\u771f\u5b9e\u63a8\u8350\u573a\u666f\uff1b<\/li>\n<li>\u8054\u5408\u8bad\u7ec3\u5bf9\u9f50\uff08\u5c0f\u7ea2\u4e66AlignRec\uff09\uff1a\u901a\u8fc7\u6df1\u5ea6\u878d\u5408\u5e73\u8861\u4e24\u8005\uff0c\u4f46\u8bad\u7ec3\u590d\u6742\u5ea6\u8f83\u9ad8\uff1b<\/li>\n<\/ul>\n<p><strong>\u7a20\u5bc6\u4e0e\u7a00\u758f\u7279\u5f81\u878d\u5408\u65b9\u6848**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u91cf\u5316\u7f16\u7801\uff08\u5feb\u624b\uff09\uff1a\u5c06\u8bed\u4e49\u7279\u5f81\u8f6c\u5316\u4e3a\u53ef\u66f4\u65b0\u7684\u8bed\u4e49ID\uff0c\u89e3\u51b3\u591a\u6a21\u6001\u8868\u5f81\u66f4\u65b0\u95ee\u9898\uff1b<\/li>\n<li>\u76f8\u4f3c\u5ea6\u5206\u6876\uff08\u6dd8\u5b9d\uff09\uff1a\u5c06\u8fde\u7eed\u76f8\u4f3c\u5ea6\u79bb\u6563\u5316\u4e3a\u56fa\u5b9a\u7ef4\u5ea6\u5411\u91cf\uff0c\u7b80\u5316\u4f7f\u7528\u65b9\u5f0f\uff1b<\/li>\n<li>\u7279\u5f81\u805a\u5408\uff08\u5c0f\u7ea2\u4e66\uff09\uff1a\u901a\u8fc7\u805a\u5408\u6a21\u5757\u5b9e\u73b0\u6df1\u5ea6\u878d\u5408\uff0c\u4fdd\u6301\u8868\u5f81\u80fd\u529b\uff1b<\/li>\n<\/ul>\n<p>\u8fd9\u4e9b\u63a2\u7d22\u8868\u660e\uff0c\u591a\u6a21\u6001\u63a8\u8350\u6b63\u5728\u4ece\u7b80\u5355\u7684\u7279\u5f81\u62fc\u63a5\u5411\u6df1\u5ea6\u8bed\u4e49\u5bf9\u9f50\u6f14\u8fdb\uff0c\u4e0d\u540c\u7684\u6280\u672f\u8def\u7ebf\u53cd\u6620\u4e86\u5bf9&quot;\u8bed\u4e49\u7eaf\u7cb9\u6027&quot;\u4e0e&quot;\u884c\u4e3a\u9002\u914d\u6027&quot;\u5e73\u8861\u7684\u4e0d\u540c\u7406\u89e3\uff0c\u4e3a\u540e\u7eed\u751f\u6210\u5f0f\u63a8\u8350\u7684\u53d1\u5c55\u5960\u5b9a\u4e86\u91cd\u8981\u57fa\u7840\u3002<\/p>\n<p><strong>1.3 \u603b\u7ed3<\/strong><\/p>\n<p>\u7efc\u4e0a\u6240\u8ff0\uff0c\u63a8\u8350\u7b97\u6cd5\u6b63\u5728\u4ece\u4f20\u7edfID\u7279\u5f81\u5411\u591a\u6a21\u6001\u5185\u5bb9\u7279\u5f81\u6f14\u8fdb\uff1a\u65e9\u671f\u7684\u6ce8\u610f\u529b\u673a\u5236\u3001\u56fe\u795e\u7ecf\u7f51\u7edc\u5960\u5b9a\u4e86\u52a8\u6001\u5efa\u6a21\u57fa\u7840\uff0c\u8fd1\u671f\u7684\u5bf9\u6bd4\u5b66\u4e60\u3001\u9884\u8bad\u7ec3\u65b9\u6cd5\u63d0\u5347\u4e86\u7279\u5f81\u8868\u5f81\u80fd\u529b\uff0c\u800c\u6700\u65b0\u7684AlignRec\u3001SimTier\u3001QARM\u7b49\u5de5\u4f5c\u5219\u76f4\u63a5\u63a2\u7d22\u4e86\u591a\u6a21\u6001\u4fe1\u606f\u5728\u63a8\u8350\u4e2d\u7684\u5e94\u7528\u3002<\/p>\n<p>\u8fd9\u4e00\u6f14\u8fdb\u8fc7\u7a0b\u4e2d\uff0c<strong>\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u6210\u4e3a\u4e86\u5173\u952e\u7684\u6280\u672f\u57fa\u7840**<\/strong>\u3002<strong>\u65e0\u8bba\u662fAlignRec\u7684BEIT-3\u4ea4\u53c9\u6ce8\u610f\u529b\u673a\u5236\uff0c\u8fd8\u662fQARM\u7684\u8bed\u4e49ID\u91cf\u5316\u7f16\u7801\uff0c\u90fd\u4f9d\u8d56\u4e8e\u9ad8\u8d28\u91cf\u7684\u591a\u6a21\u6001\u7279\u5f81\u8868\u793a\u3002\u540c\u65f6\uff0c\u4e1a\u754c\u5728\u591a\u6a21\u6001\u8868\u5f81\u5bf9\u9f50\u7b56\u7565\u4e0a\u5f62\u6210\u4e86\u4e0d\u540c\u7684\u6280\u672f\u8def\u7ebf\uff0c\u53cd\u6620\u4e86\u5bf9&quot;\u8bed\u4e49\u7eaf\u7cb9\u6027&quot;\u4e0e&quot;\u884c\u4e3a\u9002\u914d\u6027&quot;\u5e73\u8861\u7684\u4e0d\u540c\u7406\u89e3\u3002\u8fd9\u4e9b\u6280\u672f\u63a2\u7d22\u548c\u8bbe\u8ba1\u601d\u8003\u4e3a\u6784\u5efa\u6709\u6548\u7684\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u63d0\u4f9b\u4e86\u91cd\u8981\u6307\u5bfc\uff0c\u4e5f\u4e3a\u540e\u7eed\u751f\u6210\u5f0f\u63a8\u8350\u7684\u53d1\u5c55\u5960\u5b9a\u4e86\u575a\u5b9e\u57fa\u7840\u3002<\/strong>\u56e0\u6b64\uff0c\u6df1\u5165\u7406\u89e3\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u7684\u53d1\u5c55\u8109\u7edc\u548c\u8bbe\u8ba1\u539f\u5219\uff0c\u5bf9\u4e8e\u6784\u5efa\u9ad8\u6548\u7684\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u81f3\u5173\u91cd\u8981\u3002**<\/p>\n<h2>\u4e8c\u3001\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u53d1\u5c55<\/h2>\n<h3><strong>2.1 \u6587\u672c\u8868\u5f81\u7684\u4e24\u79cd\u8303\u5f0f<\/strong><\/h3>\n<p>\u968f\u7740\u5927\u8bed\u8a00\u6a21\u578b\u7684\u53d1\u5c55\uff0c\u6587\u672c\u8868\u5f81\u6280\u672f\u5f62\u6210\u4e86\u4e24\u79cd\u4e3b\u8981\u7684\u6280\u672f\u8def\u7ebf\uff0c\u5b83\u4eec\u5728\u67b6\u6784\u8bbe\u8ba1\u548c\u8bad\u7ec3\u7b56\u7565\u4e0a\u5b58\u5728\u663e\u8457\u5dee\u5f02\uff1a<\/p>\n<h3>2.1.1 Encoder-only\u67b6\u6784\u8868\u5f81<\/h3>\n<p>Encoder-only\u67b6\u6784\u7684\u8868\u5f81\u65b9\u5f0f\u662f\u6587\u672c\u8868\u5f81\u5982BERT\u3001GTE\u7684\u4e3b\u6d41\u65b9\u6cd5\u3002<\/p>\n<p><strong>\u67b6\u6784\u7279\u70b9**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u91c7\u7528\u53cc\u5411\u6ce8\u610f\u529b\u673a\u5236\uff0c\u6bcf\u4e2atoken\u90fd\u80fd\u5173\u6ce8\u5230\u5e8f\u5217\u4e2d\u7684\u6240\u6709\u5176\u4ed6token\uff1b<\/li>\n<li>\u5929\u7136\u9002\u5408\u6587\u672c\u7406\u89e3\u548c\u8868\u5f81\u5b66\u4e60\u4efb\u52a1\uff1b<\/li>\n<\/ul>\n<p><strong>\u5168\u5c40\u7279\u5f81\u63d0\u53d6\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<ul>\n<li>CLS Token\u65b9\u5f0f\uff1a\u4f7f\u7528\u5e8f\u5217\u5f00\u5934\u7684[CLS] token\u4f5c\u4e3a\u5168\u5c40\u8868\u5f81\uff1b<\/li>\n<li>\u5e73\u5747\u6c60\u5316\u65b9\u5f0f\uff1a\u53bb\u9664CLS token\u540e\uff0c\u5bf9\u5269\u4f59token\u7279\u5f81\u8fdb\u884c\u5e73\u5747\u6c60\u5316\uff1b<\/li>\n<\/ul>\n<p><strong>\u5178\u578b\u5de5\u4f5c**<\/strong>\uff1a**<\/p>\n<ul>\n<li>GTE[12]\u901a\u8fc7\u201c<strong>\u5927\u89c4\u6a21\u65e0\u76d1\u7763\u5bf9\u6bd4\u9884\u8bad\u7ec3+\u591a\u4efb\u52a1\u76d1\u7763\u5fae\u8c03<\/strong>\u201d\u7684\u4e24\u9636\u6bb5\u9012\u8fdb\u5f0f\u8bad\u7ec3\uff0c\u7ed3\u5408<strong>\u6539\u8fdb\u7684\u53cc\u5411\u5bf9\u6bd4\u635f\u5931\u4e0e\u52a8\u6001\u6570\u636e\u91c7\u6837<\/strong>\uff0c\u5728\u65e0\u9700\u79c1\u6709\u6570\u636e\u548c\u4efb\u52a1\u63d0\u793a\u4e0b\u5b9e\u73b0\u901a\u7528\u6587\u672c\u4e0e\u4ee3\u7801\u8868\u793a\u3002<\/li>\n<\/ul>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143520295.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>GTE<\/p>\n<h3>2.1.2 Decoder-only\u67b6\u6784\u8868\u5f81<\/h3>\n<p>\u968f\u7740LLM\u7684\u5feb\u901f\u53d1\u5c55\uff0c\u4e3a\u5728\u8868\u5f81\u6280\u672f\u4e0a\u6709\u6548\u5229\u7528LLM\u5f3a\u5927\u7684\u4e16\u754c\u77e5\u8bc6\uff0c\u4ee5LLM2Vec[13]\u4e3a\u4ee3\u8868\u7684Decoder-only\u6587\u672c\u8868\u5f81\u5de5\u4f5c\u5f00\u59cb\u76db\u884c\uff0cGTE\u540c\u6837\u53d1\u5e03\u4e86\u57fa\u4e8eqwen\u7684decoder-only\u7c7b\u7cfb\u5217\u6a21\u578b\u3002<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p>LLM2Vec<\/p>\n<p><strong>\u67b6\u6784\u9002\u914d**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u6ce8\u610f\u529b\u673a\u5236\u6539\u9020\uff1a\u5c06\u539f\u59cb\u7684\u5355\u5411\u6ce8\u610f\u529b\uff08\u56e0\u679c\u63a9\u7801\uff09\u6539\u4e3a\u53cc\u5411\u6ce8\u610f\u529b\uff08\u63a9\u7801\u77e9\u9635\u5168\u4e3a1\uff09\uff1b<\/li>\n<li>\u4f7f\u9884\u8bad\u7ec3\u7684\u751f\u6210\u5f0f\u6a21\u578b\u80fd\u591f\u8fdb\u884c\u53cc\u5411\u4fe1\u606f\u4ea4\u4e92\uff1b<\/li>\n<\/ul>\n<p><strong>\u5168\u5c40\u7279\u5f81\u63d0\u53d6\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<ul>\n<li>EOS Pooling\uff1a\u53d6\u5e8f\u5217\u6700\u540e\u7684EOS token\u7279\u5f81\u4f5c\u4e3a\u5168\u5c40\u8868\u5f81 \uff08\u5728\u5355\u5411\u6ce8\u610f\u529b\u4e2d\uff0c\u53ea\u6709\u6700\u540e\u4e00\u4e2atoken\u80fd\u5173\u6ce8\u5230\u524d\u9762\u6240\u6709token\u7684\u4fe1\u606f\uff09\uff1b<\/li>\n<li>Mean Pooling\uff1a\u5bf9\u6240\u6709token\u7279\u5f81\u8fdb\u884c\u5e73\u5747\u6c60\u5316\uff08cls\u53bb\u9664\u4e0e\u5426\u5f71\u54cd\u4e0d\u5927\uff09\uff1b<\/li>\n<li>Weighted Mean Pooling\uff1a\u57fa\u4e8e\u4f4d\u7f6e\u4fe1\u606f\u7684\u52a0\u6743\u5e73\u5747 \uff08\u5e8f\u5217\u4e2d\u8d8a\u9760\u540e\u7684token\u5173\u6ce8\u5230\u7684\u5168\u5c40\u4fe1\u606f\u8d8a\u591a\uff0c\u56e0\u6b64\u7ed9\u4e88\u66f4\u9ad8\u6743\u91cd\uff09\uff1b<\/li>\n<\/ul>\n<p><strong>\u8bad\u7ec3\u65b9\u6cd5\u521b\u65b0**<\/strong>\uff1a**<\/p>\n<p><strong>1. MNTP(**<\/strong>masked next token prediction)<strong>**\uff1a<\/strong><\/p>\n<ul>\n<li>\u7c7b\u4f3cBERT\u7684\u5b8c\u5f62\u586b\u7a7a\u4efb\u52a1\uff0c\u4f46\u9002\u914ddecoder\u67b6\u6784\uff1b<\/li>\n<li>\u5173\u952e\u5dee\u5f02\uff1aBERT\u4f7f\u7528\u88ab\u63a9\u7801\u4f4d\u7f6e\u7684\u9884\u6d4b\u7ed3\u679c\u8ba1\u7b97\u635f\u5931\uff0c\u800cdecoder\u4f7f\u7528\u4e0a\u4e00\u4e2a\u8bcd\u4f4d\u7f6e\u7684\u9884\u6d4b\u7ed3\u679c\uff08\u56e0\u4e3adecoder\u7684\u9884\u6d4b\u8f93\u51fa\u5bf9\u5e94\u4e0b\u4e00\u4e2a\u8bcd\uff09\uff1b<\/li>\n<\/ul>\n<p><strong>2.<\/strong> <strong>\u65e0\u76d1\u7763\u5bf9\u6bd4\u5b66\u4e60**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>\u95ee\u9898\u80cc\u666f\uff1a\u6a21\u578b\u4e3b\u8981\u5b66\u4e60word-level\u7279\u5f81\uff0c\u7f3a\u4e4fsentence-level\u7684\u8868\u5f81\u80fd\u529b\uff1b<\/p>\n<\/li>\n<li>\n<p>\u89e3\u51b3\u65b9\u6848\uff1a<\/p>\n<\/li>\n<li>\n<p>\u5bf9\u540c\u4e00\u53e5\u5b50\u8fdb\u884c\u4e24\u6b21\u968f\u673adropout\uff0c\u751f\u6210\u4e24\u4e2a\u4e0d\u540c\u7684\u8868\u5f81<\/p>\n<\/li>\n<li>\n<p>\u5c06\u540c\u4e00\u53e5\u5b50\u7684\u4e24\u4e2a\u8868\u5f81\u4f5c\u4e3a\u6b63\u6837\u672c\u5bf9<\/p>\n<\/li>\n<li>\n<p>\u4e0d\u540c\u53e5\u5b50\u95f4\u7684\u8868\u5f81\u4f5c\u4e3a\u8d1f\u6837\u672c\u5bf9<\/p>\n<\/li>\n<li>\n<p>\u901a\u8fc7\u5bf9\u6bd4\u5b66\u4e60\u589e\u5f3a\u53e5\u5b50\u7ea7\u522b\u7684\u8868\u5f81\u80fd\u529b<\/p>\n<\/li>\n<\/ul>\n<h3>2.1.3 Qwen3 Embedding(SOTA)<\/h3>\n<p>\u4f5c\u4e3a\u5f53\u524d\u6587\u672c\u8868\u5f81\u9886\u57df\u7684\u4ee3\u8868\u6027\u5de5\u4f5c\uff0cQwen3 Embedding[14]\u5c55\u73b0\u4e86decoder-only\u67b6\u6784\u7684\u6700\u65b0\u53d1\u5c55\u6c34\u5e73\uff1a<\/p>\n<p><strong>\u53cc\u6a21\u578b\u8bbe\u8ba1**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>Embedding\u6a21\u578b\uff1a\u91c7\u7528decoder-only\u67b6\u6784\uff0c\u53d6EOS token\u4f5c\u4e3a\u6587\u672c\u8868\u5f81\uff0c\u8bad\u7ec3\u7b56\u7565\u9075\u5faaGTE\u7684\u591a\u9636\u6bb5\u5bf9\u6bd4\u5b66\u4e60\u8303\u5f0f\uff1b<\/p>\n<\/li>\n<li>\n<p>Ranking\u6a21\u578b\uff1a\u4fdd\u7559\u539f\u59cbLLM\u67b6\u6784\uff0c\u901a\u8fc7\u7279\u5b9ainstruct\u8bad\u7ec3\u8f93\u51fayes\/no\u5224\u65ad\u7ed3\u679c\uff1b<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u4e09\u9636\u6bb5\u8bad\u7ec3\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p><strong>Stage 1 - \u5927\u89c4\u6a21\u6570\u636e\u5408\u6210\uff08\u4ec5\u8bad\u7ec3Embedding\u6a21\u578b\uff09**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>\u6570\u636e\u89c4\u6a21\uff1a\u901a\u8fc7\u4e8c\u9636\u6bb5\u6570\u636e\u5408\u6210\u751f\u621015kw\u5bf9\u8bad\u7ec3\u6570\u636e\uff1b<\/p>\n<\/li>\n<li>\n<p>\u914d\u7f6e\u751f\u6210\u9636\u6bb5\uff1a<\/p>\n<ul>\n<li>\n<p>\u4ee5Qwen3\u9884\u8bad\u7ec3\u8bed\u6599\u5e93\u4e3a\u6587\u6863\u6e90\uff1b<\/p>\n<\/li>\n<li>\n<p>\u4e3a\u6bcf\u4e2a\u6587\u6863\u5408\u6210\u89d2\u8272\uff08Character\uff09\u3001\u95ee\u9898\u7c7b\u578b\uff08Question Type\uff0c\u5982\u5173\u952e\u8bcd\u3001\u4e8b\u5b9e\u3001\u603b\u7ed3\u7b49\uff09\u548c\u96be\u5ea6\u7b49\u7ea7\uff08Difficulty\uff0c\u5982\u9ad8\u4e2d\u3001\u5927\u5b66\u3001\u535a\u58eb\u7ea7\uff09\uff1b<\/p>\n<\/li>\n<li>\n<p>\u89d2\u8272\u5019\u9009\u6765\u81eaPersona Hub\uff08Ge et al., 2024\uff09\uff0c\u901a\u8fc7\u68c0\u7d22\u6a21\u578b\u9009\u51fa\u4e0e\u6587\u6863\u6700\u76f8\u5173\u7684\u4e94\u4e2a\u89d2\u8272\uff0c\u589e\u5f3a\u67e5\u8be2\u591a\u6837\u6027\uff0c\u91c7\u7528JSON\u683c\u5f0f\u8f93\u51fa\uff0c\u786e\u4fdd\u7ed3\u6784\u5316\u5904\u7406\uff1b<\/p>\n<\/li>\n<\/ul>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143520802.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<\/li>\n<li>\n<p><strong>\u67e5\u8be2\u751f\u6210\u9636\u6bb5**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>\u6839\u636e\u914d\u7f6e\u7ed3\u679c\uff0c\u7ed3\u5408\u6307\u5b9a\u7684\u67e5\u8be2\u7c7b\u578b\u3001\u957f\u5ea6\u3001\u96be\u5ea6\u548c\u8bed\u8a00\u751f\u6210\u76f8\u5173\u67e5\u8be2\uff1b<\/p>\n<\/li>\n<li>\n<p>\u8986\u76d6\u68c0\u7d22\u3001\u8bed\u4e49\u76f8\u4f3c\u6027\u7b49\u591a\u79cd\u4efb\u52a1\u7c7b\u578b\u548c\u591a\u8bed\u8a00\u573a\u666f\uff1b<\/p>\n<\/li>\n<\/ul>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143521350.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<\/li>\n<\/ul>\n<p><strong>Stage 2 - \u9ad8\u8d28\u91cf\u6570\u636e\u7cbe\u70bc\uff08\u53cc\u6a21\u578b\u8054\u5408\u8bad\u7ec3\uff09**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u6570\u636e\u7b5b\u9009\uff1a\u4eceStage 1\u6570\u636e\u4e2d\u7b5b\u9009\u4f59\u5f26\u76f8\u4f3c\u5ea6&gt;0.7\u7684\u6837\u672c\uff0c\u5f97\u52301.2kw\u5bf9\u9ad8\u8d28\u91cf\u6570\u636e\uff0c\u5e76\u8865\u5145700\u4e07\u5bf9\u5f00\u6e90\u6570\u636e\uff1b<\/li>\n<li>\u5dee\u5f02\u5316\u8bad\u7ec3\uff1a\n<ul>\n<li>Embedding\u6a21\u578b\uff1a\u91c7\u7528\u5bf9\u6bd4\u5b66\u4e60\uff0c\u4f7f\u7528InfoNCE\u635f\u5931\u51fd\u6570\uff1b<\/li>\n<li>Ranking\u6a21\u578b\uff1a\u91c7\u7528\u76d1\u7763\u5fae\u8c03\uff08SFT\uff09\uff0c\u4f7f\u7528SFT\u635f\u5931\uff0c\u6807\u7b7e\u4e3ayes\/no\u4e8c\u5206\u7c7b\uff1b<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p><strong>Stage 3 - \u6a21\u578b\u878d\u5408\u4f18\u5316\uff08\u53cc\u6a21\u578b\u6743\u91cd\u878d\u5408\uff09**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u5229\u7528SLERP\uff08\u7403\u9762\u7ebf\u6027\u63d2\u503c\uff09\u6280\u672f\u5bf9\u591a\u4e2ackpt\u8fdb\u884c\u6743\u91cd\u878d\u5408\uff1b<\/li>\n<li>\u589e\u5f3a\u6a21\u578b\u5728\u4e0d\u540c\u6570\u636e\u5206\u5e03\u4e0b\u7684\u9c81\u68d2\u6027\u548c\u6cdb\u5316\u80fd\u529b\uff1b<\/li>\n<\/ul>\n<p><strong>\u6280\u672f\u521b\u65b0\u70b9**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u7cfb\u7edf\u5316\u6570\u636e\u5408\u6210\uff1a\u901a\u8fc7\u89d2\u8272-\u4efb\u52a1-\u96be\u5ea6\u7684\u4e09\u7ef4\u914d\u7f6e\u751f\u6210\u9ad8\u8d28\u91cf\u8bad\u7ec3\u6570\u636e\uff1b<\/li>\n<li>\u53cc\u6a21\u578b\u534f\u540c\uff1aEmbedding\u548cRanking\u6a21\u578b\u5206\u5de5\u660e\u786e\uff0c\u5404\u81ea\u4f18\u5316\u7279\u5b9a\u4efb\u52a1\uff1b<\/li>\n<li>\u6743\u91cd\u878d\u5408\u7b56\u7565\uff1a\u901a\u8fc7SLERP\u6280\u672f\u63d0\u5347\u6a21\u578b\u9c81\u68d2\u6027\uff1b<\/li>\n<\/ul>\n<h3><strong>2.2 \u89c6\u89c9\u8868\u5f81\u7684\u6838\u5fc3\u65b9\u6cd5<\/strong><\/h3>\n<p>\u89c6\u89c9\u8868\u5f81\u6280\u672f\u4e3b\u8981\u56f4\u7ed5\u5982\u4f55\u4ece\u56fe\u50cf\u6570\u636e\u4e2d\u63d0\u53d6\u9ad8\u5c42\u8bed\u4e49\u7279\u5f81\u5c55\u5f00\uff0c\u5f62\u6210\u4ee5\u4e0b\u4e3b\u6d41\u8303\u5f0f\uff1a<\/p>\n<h3>2.2.1 CNN\u67b6\u6784\u8868\u5f81\uff08\u4ee5ResNet\u4e3a\u4ee3\u8868\uff09<\/h3>\n<p><strong>\u67b6\u6784\u7279\u70b9**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>\u901a\u8fc7\u5377\u79ef\u5c42\u9010\u7ea7\u63d0\u53d6\u5c40\u90e8\u7279\u5f81\uff0c\u5f62\u6210\u7a7a\u95f4\u5c42\u6b21\u5316\u8868\u5f81\uff1b<\/p>\n<\/li>\n<li>\n<p>\u6df1\u5c42\u7f51\u7edc\u901a\u8fc7\u6b8b\u5dee\u8fde\u63a5\u7f13\u89e3\u68af\u5ea6\u6d88\u5931\u95ee\u9898\uff1b<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u5168\u5c40\u7279\u5f81\u63d0\u53d6\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u5168\u5c40\u5e73\u5747\u6c60\u5316\uff08GAP\uff09\uff1a\u5bf9\u6700\u7ec8\u5377\u79ef\u5c42\u7279\u5f81\u56fe\u8fdb\u884c\u7a7a\u95f4\u7ef4\u5ea6\u5e73\u5747\u6c60\u5316\uff1b<\/li>\n<li>\u6ce8\u610f\u529b\u6c60\u5316\uff1a\u901a\u8fc7\u53ef\u5b66\u4e60\u6743\u91cd\u805a\u7126\u5173\u952e\u533a\u57df\u7279\u5f81\uff1b<\/li>\n<\/ul>\n<p><strong>\u8bad\u7ec3\u4f18\u5316**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u4e3b\u8981\u91c7\u7528\u6709\u76d1\u7763\u5206\u7c7b\u9884\u8bad\u7ec3\uff0c\u5229\u7528ImageNet\u7b49\u5927\u89c4\u6a21\u6570\u636e\u96c6\uff1b<\/li>\n<li>\u901a\u8fc7\u5bf9\u6bd4\u5b66\u4e60\u589e\u5f3a\u7279\u5f81\u5224\u522b\u6027\uff08\u5982SupCon\u635f\u5931\u51fd\u6570\uff09\uff1b<\/li>\n<\/ul>\n<h3>2.2.2 Transformer\u67b6\u6784\u8868\u5f81\uff08\u4ee5ViT\u4e3a\u4ee3\u8868\uff09<\/h3>\n<p><strong>\u67b6\u6784\u9002\u914d**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>\u56fe\u50cf\u5206\u5757\u5d4c\u5165\uff1a\u5c06\u56fe\u50cf\u5206\u5272\u4e3a\u56fa\u5b9a\u5927\u5c0f\u5757\uff0c\u7ebf\u6027\u6295\u5f71\u4e3a\u5e8f\u5217\uff1b<\/p>\n<\/li>\n<li>\n<p>\u4f4d\u7f6e\u7f16\u7801\u6ce8\u5165\uff1a\u6dfb\u52a0\u53ef\u5b66\u4e60\u4f4d\u7f6e\u7f16\u7801\u4fdd\u7559\u7a7a\u95f4\u4fe1\u606f\uff1b<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143522151.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>VIT[16]<\/p>\n<\/li>\n<\/ul>\n<p><strong>\u5168\u5c40\u7279\u5f81\u63d0\u53d6\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<ul>\n<li><strong>Class Token\u65b9\u5f0f**<\/strong>\uff1a**\u5728\u5e8f\u5217\u9996\u6dfb\u52a0\u53ef\u5b66\u4e60[class] token\u4f5c\u4e3a\u5168\u5c40\u8868\u5f81\uff1b<\/li>\n<li><strong>\u5e73\u5747\u6c60\u5316\u65b9\u5f0f**<\/strong>\uff1a**\u5bf9\u6240\u6709\u56fe\u50cf\u5757\u7279\u5f81\u8fdb\u884c\u5e73\u5747\u805a\u5408\uff1b<\/li>\n<\/ul>\n<p><strong>\u8bad\u7ec3\u65b9\u6cd5\u521b\u65b0**<\/strong>\uff1a**<\/p>\n<ol>\n<li>\u63a9\u7801\u56fe\u50cf\u5efa\u6a21\uff08MIM\uff09\uff1a<\/li>\n<\/ol>\n<ul>\n<li>\n<p>\u968f\u673a\u63a9\u7801\u56fe\u50cf\u5757\uff0c\u901a\u8fc7\u53ef\u89c1\u5757\u9884\u6d4b\u63a9\u7801\u5757\u7279\u5f81\uff1b<\/p>\n<\/li>\n<li>\n<p>\u7c7b\u4f3cMAE\u3001BEiT\u7b49\u65b9\u6cd5\u663e\u8457\u63d0\u5347\u8868\u5f81\u80fd\u529b\uff1b<\/p>\n<\/li>\n<li>\n<p>\u81ea\u76d1\u7763\u5bf9\u6bd4\u5b66\u4e60\uff1a<\/p>\n<\/li>\n<li>\n<p>\u5bf9\u540c\u4e00\u56fe\u50cf\u8fdb\u884c\u4e0d\u540c\u6570\u636e\u589e\u5f3a\u751f\u6210\u6b63\u6837\u672c\u5bf9\uff1b<\/p>\n<\/li>\n<li>\n<p>\u91c7\u7528InfoNCE\u635f\u5931\u62c9\u8fd1\u6b63\u6837\u672c\u8ddd\u79bb\uff0c\u63a8\u8fdc\u8d1f\u6837\u672c\uff1b<\/p>\n<\/li>\n<\/ul>\n<h3>2.2.3 \u4e24\u79cd\u89c6\u89c9\u8868\u5f81\u8303\u5f0f\u5bf9\u6bd4<\/h3>\n<table>\n<thead>\n<tr>\n<th>\u7279\u5f81\u7ef4\u5ea6<\/th>\n<th>CNN\u67b6\u6784<\/th>\n<th>Transformer\u67b6\u6784<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u8868\u5f81\u80fd\u529b<\/td>\n<td>\u5728\u5c0f\u6570\u636e\u96c6\u4e0a\u8868\u73b0\u7a33\u5b9a<\/td>\n<td><strong>\u5927\u89c4\u6a21\u9884\u8bad\u7ec3\u4e0b\u6548\u679c\u66f4\u4f18<\/strong><\/td>\n<\/tr>\n<tr>\n<td>\u7279\u5f81\u63d0\u53d6\u65b9\u5f0f<\/td>\n<td>\u5c40\u90e8\u5230\u5168\u5c40<\/td>\n<td>\u5168\u5c40\u6ce8\u610f\u529b\u4ea4\u4e92<\/td>\n<\/tr>\n<tr>\n<td>\u4f4d\u7f6e\u4fe1\u606f\u5904\u7406<\/td>\n<td>\u5377\u79ef\u56fa\u6709\u611f\u5e94<\/td>\n<td>\u9700\u663e\u5f0f\u4f4d\u7f6e\u7f16\u7801<\/td>\n<\/tr>\n<tr>\n<td>\u6570\u636e\u4f9d\u8d56\u6027<\/td>\n<td>\u76f8\u5bf9\u4f4e\u6570\u636e\u9700\u6c42<\/td>\n<td>\u9700\u5927\u89c4\u6a21\u9884\u8bad\u7ec3<\/td>\n<\/tr>\n<tr>\n<td>\u8ba1\u7b97\u6548\u7387<\/td>\n<td>\u4f18\u5316\u7a0b\u5ea6\u9ad8<\/td>\n<td>\u9ad8\u5206\u8fa8\u7387\u8ba1\u7b97\u6602\u8d35<\/td>\n<\/tr>\n<tr>\n<td>\u5f53\u524d\u8d8b\u52bf<\/td>\n<td>\u5728\u7279\u5b9a\u573a\u666f\u4ecd\u6709\u4ef7\u503c<\/td>\n<td><strong>\u4e3b\u6d41\u89c6\u89c9\u8868\u5f81\u65b9\u6cd5<\/strong><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6587\u672c\u8868\u5f81\u4e0e\u89c6\u89c9\u8868\u5f81\u6280\u672f\u7684\u53d1\u5c55\u4e3a\u591a\u6a21\u6001\u878d\u5408\u5960\u5b9a\u4e86\u575a\u5b9e\u57fa\u7840\uff0c\u4e24\u79cd\u6a21\u6001\u7684\u8868\u5f81\u65b9\u6cd5\u5728\u8bbe\u8ba1\u7406\u5ff5\u548c\u6280\u672f\u5b9e\u73b0\u4e0a\u65e2\u6709\u76f8\u4f3c\u6027\u53c8\u6709\u4e92\u8865\u6027\uff0c\u4e3a\u540e\u7eed\u7684\u8de8\u6a21\u6001\u5bf9\u9f50\u548c\u878d\u5408\u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u6280\u672f\u9009\u62e9\u3002<\/p>\n<h3><strong>2.3 \u56fe\u6587\u591a\u6a21\u6001\u8868\u5f81\u6838\u5fc3\u65b9\u6cd5<\/strong><\/h3>\n<p>\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u6839\u636e\u4e0d\u540c\u6a21\u6001\u4fe1\u606f\u7684\u878d\u5408\u65b9\u5f0f\uff0c\u4e3b\u8981\u5206\u4e3a\u534f\u540c\u8868\u5f81\u548c\u8054\u5408\u8868\u5f81\u4e24\u5927\u7c7b\u6280\u672f\u8def\u7ebf\uff1a<\/p>\n<h3>2.3.1 \u534f\u540c\u8868\u5f81\u65b9\u6cd5\uff1a\u53cc\u5854\u67b6\u6784\u7684\u591a\u6a21\u6001\u5bf9\u9f50<\/h3>\n<p>\u534f\u540c\u8868\u5f81\u65b9\u6cd5\u91c7\u7528\u72ec\u7acb\u7684\u7f16\u7801\u5668\u5206\u522b\u5904\u7406\u4e0d\u540c\u6a21\u6001\u4fe1\u606f\uff0c\u901a\u8fc7\u5bf9\u6bd4\u5b66\u4e60\u7b49\u65b9\u5f0f\u5b9e\u73b0\u8de8\u6a21\u6001\u5bf9\u9f50\u3002<\/p>\n<h4><strong>2.3.1.1<\/strong> <strong>CLIP**<\/strong>[17]<strong>**\uff1a\u5960\u5b9a\u53cc\u5854\u5bf9\u6bd4\u5b66\u4e60\u57fa\u7840<\/strong><\/h4>\n<p><strong>\u67b6\u6784\u8bbe\u8ba1**<\/strong>\uff1a**<\/p>\n<ul>\n<li><strong>\u53cc\u5854\u7ed3\u6784**<\/strong>\uff1a**\u56fe\u50cf\u7f16\u7801\u5668\uff08ViT\/ResNet\uff09\u548c\u6587\u672c\u7f16\u7801\u5668\uff08Transformer\uff09\u72ec\u7acb\u5904\u7406\u5404\u81ea\u6a21\u6001\uff1b<\/li>\n<li><strong>\u7279\u5f81\u5bf9\u9f50**<\/strong>\uff1a**\u901a\u8fc7\u7ebf\u6027\u6295\u5f71\u5c42\u5c06\u4e0d\u540c\u6a21\u6001\u7279\u5f81\u6620\u5c04\u5230\u7edf\u4e00\u8bed\u4e49\u7a7a\u95f4\uff1b<\/li>\n<\/ul>\n<p><strong>\u8bad\u7ec3\u673a\u5236**<\/strong>\uff1a**<\/p>\n<ul>\n<li><strong>InfoNCE\u635f\u5931**<\/strong>\uff1a**n\u4e2a\u56fe\u50cf\u4e0en\u4e2a\u6587\u672c\u5206\u522b\u7f16\u7801\u540e\u8ba1\u7b97\u4f59\u5f26\u76f8\u4f3c\u5ea6\u77e9\u9635\uff1b<\/li>\n<li><strong>\u6e29\u5ea6\u7f29\u653e**<\/strong>\uff1a**\u901a\u8fc7\u53ef\u5b66\u4e60\u6e29\u5ea6\u53c2\u6570\u63d0\u5347\u76f8\u4f3c\u5ea6\u533a\u5206\u5ea6\uff1b<\/li>\n<li><strong>\u5bf9\u6bd4\u5b66\u4e60**<\/strong>\uff1a**\u5bf9\u89d2\u7ebf\u5143\u7d20\u4e3a\u6b63\u6837\u672c\uff0c\u5176\u4f59\u4e3a\u8d1f\u6837\u672c\u8fdb\u884c\u4ea4\u53c9\u71b5\u4f18\u5316\uff1b<\/li>\n<\/ul>\n<p><strong>\u7279\u70b9**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\n<p>\u63a8\u7406\u65f6\uff0cCLIP\u4e3b\u8981\u7528\u6765\u505a\u7279\u5f81\u63d0\u53d6\uff0c\u5728\u56fe\u7247\u5206\u7c7b\u4efb\u52a1\u4e0a\u7684\u7528\u6cd5\u5219\u662f\uff1avit\u7b97\u56fe\u5411\u91cf\u540e\uff0c\u6587\u672c\u7f16\u7801\u5668\u7b97\u6807\u7b7e\u6587\u672c\u5bf9\u5e94\u8bcd\u5411\u91cf\uff0c\u8ba1\u7b97\u4f59\u5f26\u76f8\u4f3c\u5ea6\u53d6\u6700\u5927\u7684\u6807\u7b7e\uff1b<\/p>\n<\/li>\n<li>\n<p>\u4f18\u52bf\uff1a<\/p>\n<ul>\n<li>\u5728\u5927\u89c4\u6a21\u56fe\u50cf-\u6587\u672c\u5bf9\u4e0a\u9884\u8bad\u7ec3\uff0c\u5c55\u73b0\u5f3a\u5927\u7684\u96f6\u6837\u672c\u80fd\u529b\uff1b<\/li>\n<li>\u6781\u5176\u9002\u5408\u5927\u89c4\u6a21\u56fe\u6587\u68c0\u7d22\u3001\u56fe\u6587\u7279\u5f81\u63d0\u53d6\u7b49\u4e0b\u6e38\u4efb\u52a1\uff1b<\/li>\n<\/ul>\n<\/li>\n<li>\n<p>\u7f3a\u70b9\uff1a\u53ea\u8ba1\u7b97Cosine Similarity\u65e0\u6cd5\u505a\u591a\u6a21\u6001\u4e4b\u95f4\u6df1\u5ea6\u878d\u5408\uff0c\u96be\u4e00\u4e9b\u7684\u4efb\u52a1\u6027\u80fd\u8f83\u5dee\uff1b<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143522726.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>CLIP<\/p>\n<\/li>\n<\/ul>\n<h4><strong>2.3.1.2<\/strong> <strong>ALBEF**<\/strong>[18]<strong>**\uff1a\u589e\u5f3a\u7684\u591a\u6a21\u6001\u878d\u5408<\/strong><\/h4>\n<p><strong>\u67b6\u6784\u6539\u8fdb**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u7279\u5f81\u91cd\u8981\u6027\u8bbe\u8ba1\uff1a\u89c6\u89c9&gt;&gt;\u6587\u672c\uff0c12\u5c42vit encoder for\u89c6\u89c9\uff0c6\u5c42transformer encoder for\u6587\u672c\uff0c6\u5c42\u591a\u6a21encoder for\u7279\u5f81\u878d\u5408\uff1b<\/li>\n<li>\u4ea4\u53c9\u6ce8\u610f\u529b\u673a\u5236\uff1a\u56fe\u50cf\u7279\u5f81\u4f5c\u4e3aQuery\u4e0e\u6587\u672c\u7279\u5f81\u7684Key-Value\u8ba1\u7b97\u6ce8\u610f\u529b\uff0c\u53cd\u4e4b\u4ea6\u7136\uff1b<\/li>\n<\/ul>\n<p><strong>\u591a\u4efb\u52a1\u8bad\u7ec3**<\/strong>\uff1a**<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p>ALBEF<\/p>\n<ul>\n<li><strong>\u56fe\u6587\u5bf9\u6bd4\u5b66\u4e60\uff08ITC\uff09**<\/strong>\uff1a**\u7c7b\u4f3cCLIP\u7684InfoNCE\u635f\u5931\uff1b<\/li>\n<li><strong>\u56fe\u6587\u5339\u914d\uff08ITM\uff09**<\/strong>\uff1a**\u4e8c\u5206\u7c7b\u4efb\u52a1\uff0c\u52a0\u5165\u96be\u8d1f\u6837\u672c\u6316\u6398\u7b56\u7565\uff1b<\/li>\n<li><strong>\u63a9\u7801\u8bed\u8a00\u5efa\u6a21\uff08MLM\uff09**<\/strong>\uff1a**\u4f20\u7edfBERT\u5f0f\u5b8c\u5f62\u586b\u7a7a\u4efb\u52a1\uff1b<\/li>\n<\/ul>\n<p><strong>\u6280\u672f\u521b\u65b0**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u501f\u9274MoCo\u7684\u52a8\u91cf\u601d\u60f3\u6784\u5efa\u52a8\u91cf\u8bad\u7ec3\u6846\u67b6\uff1b<\/li>\n<li>\u901a\u8fc7\u52a8\u91cf\u6a21\u578b\u84b8\u998f\u63d0\u5347\u6cdb\u5316\u80fd\u529b\uff0c\u5728VQA\u7b49\u4efb\u52a1\u4e2d\u51c6\u786e\u7387\u63d0\u53473.84%\uff1b<\/li>\n<\/ul>\n<h4><strong>2.3.1.3 \u5176\u4ed6\u534f\u540c\u8868\u5f81\u5de5\u4f5c<\/strong><\/h4>\n<p>\u5728CLIP\u548cALBEF\u7684\u57fa\u7840\u4e0a\uff0c\u8fd8\u6d8c\u73b0\u51faDeCLIP\u3001FLAVA\u3001CoCa\u7b49\u6539\u8fdb\u5de5\u4f5c\u3002DeCLIP\u901a\u8fc7\u591a\u4e2a\u9884\u8bad\u7ec3\u4efb\u52a1\uff08\u56fe\u50cf\u81ea\u76d1\u7763\u3001\u6587\u672c\u81ea\u76d1\u7763\u3001\u8de8\u6a21\u6001\u81ea\u76d1\u7763\uff09\u7528\u66f4\u5c11\u6570\u636e\u8fbe\u5230\u76f8\u8fd1\u6548\u679c\uff1bFLAVA\u5f15\u5165\u66f4\u591a\u9884\u8bad\u7ec3\u4efb\u52a1\u5982MIM\u548cMMM\u635f\u5931\uff1bCoCa\u5728ALBEF\u57fa\u7840\u4e0a\u52a0\u5165\u89e3\u7801\u5668\u589e\u5f3a\u751f\u6210\u80fd\u529b\u3002\u8fd9\u4e9b\u5de5\u4f5c\u5728\u534f\u540c\u8868\u5f81\u6846\u67b6\u4e0b\u4e0d\u65ad\u4f18\u5316\u8bad\u7ec3\u7b56\u7565\u548c\u4efb\u52a1\u8bbe\u8ba1\uff0c\u4f46\u57fa\u672c\u67b6\u6784\u601d\u8def\u4fdd\u6301\u4e00\u81f4\uff0c\u6b64\u5904\u4e0d\u518d\u4e00\u4e00\u5c55\u5f00\u3002<\/p>\n<h3>2.3.2 \u8054\u5408\u8868\u5f81\u65b9\u6cd5\uff1a\u7edf\u4e00\u67b6\u6784\u7684\u6df1\u5ea6\u878d\u5408<\/h3>\n<p>\u8054\u5408\u8868\u5f81\u65b9\u6cd5\u4f7f\u7528\u7edf\u4e00\u7684\u6a21\u578b\u67b6\u6784\u540c\u65f6\u5904\u7406\u591a\u79cd\u6a21\u6001\u4fe1\u606f\uff0c\u5b9e\u73b0\u66f4\u6df1\u5c42\u6b21\u7684\u8de8\u6a21\u6001\u7406\u89e3\u3002<\/p>\n<h4><strong>2.3.2.1 \u65e9\u671f\u63a2\u7d22\u5de5\u4f5c<\/strong><\/h4>\n<p>\u65e9\u671f\u7684\u8054\u5408\u8868\u5f81\u63a2\u7d22\u5305\u62ecVLMO\u3001BLIP\u3001BEiT-3\u7b49\u5de5\u4f5c\uff0c\u5b83\u4eec\u7684\u5171\u540c\u7279\u70b9\u662f\u4ece\u53cc\u5854\u7ed3\u6784\u5411\u7edf\u4e00\u67b6\u6784\u53d1\u5c55\uff1aVLMO\u901a\u8fc7Multiway Transformer\u5b9e\u73b0\u6a21\u6001\u5171\u4eab\uff1bBLIP\u63d0\u51fa\u7f16\u7801\u5668-\u89e3\u7801\u5668\u6df7\u5408\u67b6\u6784\u5e76\u5f15\u5165bootstrapping\u6570\u636e\u6e05\u6d17\uff1bBEiT-3\u7ed3\u5408\u591a\u79cd\u65b9\u6cd5\u5f62\u6210\u7edf\u4e00\u5927\u6a21\u578b\u3002\u8fd9\u4e9b\u5de5\u4f5c\u63a8\u52a8\u4e86\u591a\u6a21\u6001\u6280\u672f\u4ece\u53cc\u5854\u534f\u540c\u5411\u7edf\u4e00\u5efa\u6a21\u7684\u8f6c\u53d8\uff0c\u4e3a\u540e\u7eed\u7684VLM\u7edf\u4e00\u8868\u5f81\u5960\u5b9a\u4e86\u91cd\u8981\u57fa\u7840\u3002<\/p>\n<h4><strong>2.3.2.2<\/strong> <strong>VLM2Vec\u8303\u5f0f**<\/strong>[19]<strong>**\uff1a\u7edf\u4e00\u591a\u6a21\u6001\u8868\u5f81\u7684\u65b0\u7a81\u7834<\/strong><\/h4>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p>VLM2Vec<\/p>\n<p><strong>\u6838\u5fc3\u7406\u5ff5**<\/strong>\uff1a** \u5c06LLM2Vec\u7684\u601d\u8def\u8fc1\u79fb\u81f3\u4e86\u591a\u6a21\u6001\u8868\u5f81\u9886\u57df\uff0c\u76f4\u63a5\u57fa\u4e8e\u9884\u8bad\u7ec3\u7684\u591a\u6a21\u6001\u5927\u6a21\u578b\u8bad\u7ec3\u5f97\u5230\u591a\u6a21\u6001\u8868\u5f81\u6a21\u578b\uff0c\u5145\u5206\u5229\u7528\u5176\u4e16\u754c\u77e5\u8bc6\u3002<\/p>\n<p><strong>\u6280\u672f\u5b9e\u73b0**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u7edf\u4e00\u8868\u5f81\u63d0\u53d6\uff1a\u76f4\u63a5\u5728\u8f93\u5165\u5e8f\u5217\u6700\u540e\u6dfb\u52a0EOS token\uff0c\u53d6VLM\u8f93\u51fa\u6700\u540e\u4e00\u4e2atoken\u4f5c\u4e3a\u6574\u4f53\u7279\u5f81\uff1b<\/li>\n<li>\u4efb\u52a1\u91cd\u6784\u7b56\u7565\uff1a\u5c06\u591a\u79cd\u591a\u6a21\u6001\u4efb\u52a1\u91cd\u65b0\u8bbe\u8ba1\u4e3a\u6392\u540d\u95ee\u9898\uff0c\u6a21\u578b\u6839\u636e\u6307\u4ee4\u5904\u7406\u67e5\u8be2\u5e76\u4ece\u5019\u9009\u4e2d\u9009\u62e9\u6b63\u786e\u7b54\u6848\uff1b<\/li>\n<\/ul>\n<p><strong>\u56db\u5927\u8bad\u7ec3\u4efb\u52a1**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u56fe\u50cf\u5206\u7c7b\uff1a\u67e5\u8be2\u4e3a\u56fe\u50cf+\u6307\u4ee4\uff0c\u76ee\u6807\u4e3a\u7c7b\u522b\u6807\u7b7e\uff1b<\/li>\n<li>\u89c6\u89c9\u95ee\u7b54\uff08VQA\uff09\uff1a\u67e5\u8be2\u4e3a\u56fe\u50cf+\u95ee\u9898\uff0c\u76ee\u6807\u4e3a\u7b54\u6848\u9009\u9879\uff1b<\/li>\n<li>\u56fe\u6587\u68c0\u7d22\uff1a\u67e5\u8be2\u548c\u76ee\u6807\u53ef\u4ee5\u662f\u6587\u672c\u3001\u56fe\u50cf\u6216\u4e24\u8005\u7ec4\u5408\uff1b<\/li>\n<li>\u89c6\u89c9\u5b9a\u4f4d\uff1a\u7a7a\u95f4\u4f4d\u7f6e\u76f8\u5173\u7684\u89c6\u89c9\u7406\u89e3\u4efb\u52a1\uff1b<\/li>\n<\/ul>\n<p><strong>\u6280\u672f\u4f18\u52bf**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u67b6\u6784\u7b80\u6d01\uff1a\u65e0\u9700\u50cfLLM2Vec\u90a3\u6837\u5bf9LLM\u672c\u8eab\u505a\u592a\u591a\u6539\u52a8\u4f7f\u5176\u5411BERT\u7684\u8bad\u7ec3\u65b9\u5f0f\u5bf9\u9f50\uff1b<\/li>\n<li>\u5bf9\u6bd4\u5b66\u4e60\u5bfc\u5411\uff1a\u5145\u5206\u5229\u7528\u591a\u6a21\u6001\u573a\u666f\u4e0b\u5bf9\u6bd4\u5b66\u4e60\u7684\u6838\u5fc3\u4f5c\u7528\uff1b<\/li>\n<li>\u77e5\u8bc6\u7ee7\u627f\uff1a\u5145\u5206\u5229\u7528VLM\u5728\u591a\u6a21\u6001\u7406\u89e3\u65b9\u9762\u7684\u9884\u8bad\u7ec3\u77e5\u8bc6\uff1b<\/li>\n<li>\u4efb\u52a1\u7edf\u4e00\uff1a\u901a\u8fc7\u7edf\u4e00\u7684\u6392\u540d\u6846\u67b6\u5904\u7406\u591a\u79cd\u591a\u6a21\u6001\u4efb\u52a1\uff1b<\/li>\n<\/ul>\n<h4><strong>2.3.2.3<\/strong> <strong>GME**<\/strong>[20]<strong>**\uff1a\u7edf\u4e00\u591a\u6a21\u6001\u8868\u5f81\u7684\u5178\u578b\u5b9e\u73b0<\/strong><\/h4>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p>GME<\/p>\n<p><strong>\u6838\u5fc3\u7406\u5ff5**<\/strong>\uff1a** \u57fa\u4e8eVLM2Vec\u8303\u5f0f\uff0c\u6784\u5efa\u7edf\u4e00\u7684\u591a\u6a21\u6001\u8868\u5f81\u6a21\u578b\uff0c\u89e3\u51b3\u4f20\u7edf\u65b9\u6cd5\u9700\u8981\u90e8\u7f72\u591a\u4e2a\u72ec\u7acb\u6a21\u578b\u6216\u4f9d\u8d56OCR\/Caption\u8f6c\u6362\u5bfc\u81f4\u7684\u4fe1\u606f\u635f\u5931\u95ee\u9898\u3002<\/p>\n<p><strong>\u6280\u672f\u5b9e\u73b0**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u7edf\u4e00\u67b6\u6784\uff1a\u4ee5MLLM\u4e3a\u5e95\u5ea7\uff0c\u4f7f\u7528\u6700\u540e\u4e00\u4e2atoken\u7684\u9690\u85cf\u72b6\u6001\u4f5c\u4e3a\u7edf\u4e00\u8868\u5f81\uff1b<\/li>\n<li>\u6570\u636e\u5408\u6210\u7b56\u7565\uff1a\u91c7\u7528Doc2Query\u65b9\u6cd5\u4ece\u7ef4\u57fa\u767e\u79d1\u5408\u6210\u767e\u4e07\u7ea7\u6df7\u5408\u6a21\u6001\u8bad\u7ec3\u6570\u636e\uff1b<\/li>\n<li>\u6307\u4ee4\u8c03\u4f18\uff1a\u901a\u8fc7\u5b9a\u5236\u5316\u6307\u4ee4\u9002\u5e94\u4e0d\u540c\u68c0\u7d22\u4efb\u52a1\u573a\u666f\uff1b<\/li>\n<\/ul>\n<p><strong>\u4e09\u7c7b\u68c0\u7d22\u4efb\u52a1**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u5355\u6a21\u6001\u641c\u7d22\uff1aT\u2192T\u3001I\u2192I\u7b49\u540c\u6a21\u6001\u68c0\u7d22\uff1b<\/li>\n<li>\u8de8\u6a21\u6001\u641c\u7d22\uff1aT\u2192I\u3001T\u2192VD\uff08\u6587\u672c\u5230\u89c6\u89c9\u6587\u6863\uff09\u3001I\u2192T\u7b49\uff1b<\/li>\n<li>\u6df7\u5408\u6a21\u6001\u641c\u7d22\uff1aIT\u2192IT\u7b49\u590d\u6742\u591a\u6a21\u6001\u7ec4\u5408\u68c0\u7d22\uff1b<\/li>\n<\/ul>\n<p><strong>\u6280\u672f\u4f18\u52bf**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u771f\u6b63\u7edf\u4e00\uff1a\u5355\u4e00\u6a21\u578b\u5904\u7406\u6240\u6709\u6a21\u6001\u7ec4\u5408\uff0c\u7a81\u7834CLIP\u7b49\u4f20\u7edf\u65b9\u6cd5\u7684\u5355\u4e00\u6a21\u6001\u5c40\u9650\uff1b<\/li>\n<li>\u6548\u679c\u9886\u5148\uff1aGME-Qwen2-VL-7B\u5728UMRB\u8bc4\u6d4b\u8fbe\u523067.44\u5e73\u5747\u5206\uff0c\u8d85\u8d8a\u6240\u6709\u57fa\u7ebf\uff1b<\/li>\n<li>\u5de5\u7a0b\u53cb\u597d\uff1a\u907f\u514d\u591a\u6a21\u578b\u90e8\u7f72\u590d\u6742\u5ea6\uff0c\u5bcc\u6587\u672c\u68c0\u7d22\u76f4\u63a5\u89c6\u89c9\u5904\u7406\u4f18\u4e8eOCR\u8f6c\u6362\uff1b<\/li>\n<li>\u5b9e\u7528\u4ef7\u503c\uff1a\u4e3aRAG\u7b49\u5e94\u7528\u63d0\u4f9b\u7edf\u4e00\u7684\u591a\u6a21\u6001\u68c0\u7d22\u89e3\u51b3\u65b9\u6848\uff1b<\/li>\n<\/ul>\n<h3>2.3.3 \u6280\u672f\u53d1\u5c55\u8d8b\u52bf\u4e0e\u5c55\u671b<\/h3>\n<p>\u4ece\u534f\u540c\u8868\u5f81\u5230\u8054\u5408\u8868\u5f81\u7684\u6f14\u8fdb\u53cd\u6620\u4e86\u591a\u6a21\u6001\u6280\u672f\u7684\u53d1\u5c55\u65b9\u5411\uff0c\u4f46\u4e24\u79cd\u8303\u5f0f\u5728\u4e0d\u540c\u573a\u666f\u4e0b\u5404\u6709\u4f18\u52a3\uff1a<\/p>\n<p><strong>\u8054\u5408\u8868\u5f81\u7684\u4f18\u52bf**<\/strong>\uff1a**\u7edf\u4e00\u67b6\u6784\u5b9e\u73b0\u6df1\u5ea6\u8de8\u6a21\u6001\u878d\u5408\uff0c\u5355\u6a21\u578b\u5904\u7406\u591a\u79cd\u4efb\u52a1\uff0c\u90e8\u7f72\u4fbf\u6377\uff0c\u4ee3\u8868\u6280\u672f\u53d1\u5c55\u4e3b\u6d41\u8d8b\u52bf\u3002<\/p>\n<p><strong>\u534f\u540c\u8868\u5f81\u7684\u6301\u7eed\u4ef7\u503c**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u7075\u6d3b\u6027\uff1a\u53ef\u7075\u6d3b\u9009\u62e9\u5f53\u524d\u6700\u5f3a\u7684\u6587\u672c\u7f16\u7801\u5668\u548c\u89c6\u89c9\u7f16\u7801\u5668\u8fdb\u884c\u7ec4\u88c5\uff1b<\/li>\n<li>\u9002\u914d\u6027\uff1a\u5728\u4e0b\u6e38\u573a\u666f\u5fae\u8c03\u65f6\uff0c\u53cc\u5854\u7ed3\u6784\u5141\u8bb8\u5bf9\u4e0d\u540c\u6a21\u6001\u7f16\u7801\u5668\u72ec\u7acb\u4f18\u5316\uff1b<\/li>\n<li>\u5b9e\u7528\u6027\uff1a\u5728\u8ba1\u7b97\u8d44\u6e90\u53d7\u9650\u6216\u5bf9\u7279\u5b9a\u6a21\u6001\u6709\u6781\u81f4\u6027\u80fd\u8981\u6c42\u65f6\u4ecd\u5177\u4f18\u52bf\uff1b<\/li>\n<\/ul>\n<p><strong>\u573a\u666f\u4f9d\u8d56\u7684\u6280\u672f\u9009\u62e9**<\/strong>\uff1a**<\/p>\n<ul>\n<li>\u901a\u7528\u573a\u666f\u4e0b\uff0c\u8054\u5408\u8868\u5f81\u5c55\u73b0\u66f4\u5f3a\u7edf\u4e00\u5efa\u6a21\u80fd\u529b\uff1b<\/li>\n<li>\u7279\u5b9a\u9886\u57df\u4e2d\uff0c\u534f\u540c\u8868\u5f81\u5728\u9488\u5bf9\u6027\u4f18\u5316\u548c\u7ec4\u4ef6\u66ff\u6362\u65b9\u9762\u4ecd\u6709\u72ec\u7279\u4ef7\u503c\uff1b<\/li>\n<li>\u5728\u9700\u8981\u4e0b\u6e38\u5fae\u8c03\u7684\u60c5\u51b5\u4e0b\uff0c\u4e24\u79cd\u8303\u5f0f\u7684\u6700\u7ec8\u8868\u73b0\u4ecd\u9700\u5177\u4f53\u9a8c\u8bc1\uff0c\u5c1a\u65e0\u7edd\u5bf9\u4f18\u52a3\uff1b<\/li>\n<\/ul>\n<p><strong>2.4 \u603b\u7ed3<\/strong><\/p>\n<p>\u901a\u8fc7\u5bf9\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u53d1\u5c55\u8109\u7edc\u7684\u68b3\u7406\uff0c\u6211\u4eec\u53ef\u4ee5\u770b\u5230\u8be5\u9886\u57df\u5448\u73b0\u51fa\u591a\u5143\u5316\u53d1\u5c55\u6001\u52bf\uff1a\u6587\u672c\u8868\u5f81\u4eceEncoder-only\u5411Decoder-only\u67b6\u6784\u6f14\u8fdb\uff0c\u89c6\u89c9\u8868\u5f81\u4eceCNN\u5411Transformer\u8fc1\u79fb\uff0c\u800c\u591a\u6a21\u6001\u8868\u5f81\u5219\u5f62\u6210\u4e86\u534f\u540c\u8868\u5f81\u548c\u8054\u5408\u8868\u5f81\u4e24\u6761\u5e76\u884c\u53d1\u5c55\u7684\u6280\u672f\u8def\u7ebf\u3002<\/p>\n<p>\u8fd9\u4e00\u6280\u672f\u6f14\u8fdb\u4e3a\u63a8\u8350\u7cfb\u7edf\u63d0\u4f9b\u4e86\u4e30\u5bcc\u7684\u591a\u6a21\u6001\u7279\u5f81\u63d0\u53d6\u9009\u62e9\uff1a<strong>\u534f\u540c\u8868\u5f81\u65b9\u6cd5<\/strong>\u5982CLIP\u63d0\u4f9b\u4e86\u6210\u719f\u7a33\u5b9a\u7684\u56fe\u6587\u5bf9\u9f50\u80fd\u529b\uff0c\u4e14\u80fd\u591f\u7075\u6d3b\u7ec4\u5408\u5f53\u524dSOTA\u7684\u5355\u6a21\u6001\u7f16\u7801\u5668\uff1b<strong>\u8054\u5408\u8868\u5f81\u65b9\u6cd5<\/strong>\u5982VLM2Vec\u3001GME\u5219\u5c55\u73b0\u4e86\u66f4\u5f3a\u7684\u8de8\u6a21\u6001\u7406\u89e3\u80fd\u529b\uff0c\u4f46\u5728\u6a21\u578b\u9009\u62e9\u548c\u4f18\u5316\u4e0a\u76f8\u5bf9\u56fa\u5316\u3002<\/p>\n<p>\u57fa\u4e8e\u8fd9\u4e00\u6280\u672f\u53d1\u5c55\u8109\u7edc\u548c\u5b9e\u9645\u4e1a\u52a1\u9700\u6c42\uff0c\u6211\u4eec\u572824\u5e74S2\u9762\u5411\u997f\u4e86\u4e48\u9996\u9875\u5e97\u94fa\u63a8\u8350\u573a\u666f\u8bbe\u8ba1\u4e86\u591a\u6a21\u6001\u63a8\u8350\u65b9\u6848\uff0c\u901a\u8fc7\u4e09\u9636\u6bb5\u8bad\u7ec3\u7b56\u7565\u7cfb\u7edf\u6027\u5730\u89e3\u51b3\u4e86\u591a\u6a21\u6001\u7279\u5f81\u9884\u8bad\u7ec3\u3001\u7528\u6237\u67e5\u8be2\u5bf9\u9f50\u4ee5\u53ca\u4e0e\u4f20\u7edfID\u7279\u5f81\u878d\u5408\u7b49\u5173\u952e\u6280\u672f\u95ee\u9898\u3002<\/p>\n<h2>\u4e09\u3001\u997f\u4e86\u4e48\u9996\u9875\u63a8\u8350\u573a\u666f\u4e0b\u7684\u591a\u6a21\u6001\u63a8\u8350\u5b9e\u8df5<\/h2>\n<p>\u5728<strong>\u997f\u4e86\u4e48\u9996\u9875\u5e97\u94fa\u63a8\u8350\u573a\u666f<\/strong>\u7684\u5b9e\u9645\u5e94\u7528\u4e2d\uff0c\u7528\u6237\u7684\u51b3\u7b56\u8fc7\u7a0b\u5f80\u5f80\u7531\u5546\u6237\/\u5546\u54c1\u7684\u591a\u6a21\u6001\u5185\u5bb9\u4fe1\u606f\u76f4\u63a5\u9a71\u52a8\u3002\u5f53\u7528\u6237\u6d4f\u89c8\u63a8\u8350\u9875\u9762\u65f6\uff0c\u5546\u6237\u56fe\u7247\u7684\u89c6\u89c9\u5448\u73b0\u3001\u6587\u5b57\u63cf\u8ff0\u7684\u5438\u5f15\u529b\u7b49\u5185\u5bb9\u7279\u5f81\u662f\u5176\u6700\u5148\u611f\u77e5\u5230\u7684\u4fe1\u606f\uff0c\u8fd9\u4e9b\u591a\u6a21\u6001\u4fe1\u606f\u5728\u5f88\u5927\u7a0b\u5ea6\u4e0a\u51b3\u5b9a\u4e86\u7528\u6237\u7684\u70b9\u51fb\u548c\u8d2d\u4e70\u884c\u4e3a\u3002<\/p>\n<p>\u7136\u800c\uff0c\u5982\u524d\u6587\u6240\u8ff0\uff0c\u4f20\u7edf\u63a8\u8350\u7cfb\u7edf\u4e3b\u8981\u4f9d\u8d56ID\u7279\u5f81\u8fdb\u884c\u5efa\u6a21\uff0c\u96be\u4ee5\u6709\u6548\u5229\u7528\u8fd9\u4e9b\u4e30\u5bcc\u7684\u5185\u5bb9\u4fe1\u606f\u3002\u7279\u522b\u662f\u5728\u672c\u5730\u751f\u6d3b\u573a\u666f\u4e2d\uff0c\u5546\u6237\u7684\u89c6\u89c9\u5c55\u793a\uff08\u5982\u5e97\u94falogo\u3001\u83dc\u54c1\u56fe\u7247\uff09\u548c\u6587\u672c\u63cf\u8ff0\uff08\u5982\u7279\u8272\u6807\u7b7e\u3001\u7528\u6237\u8bc4\u4ef7\uff09\u5bf9\u7528\u6237\u9009\u62e9\u5177\u6709\u91cd\u8981\u5f71\u54cd\uff0c\u4f46\u8fd9\u4e9b\u4fe1\u606f\u5728\u4f20\u7edfID\u5efa\u6a21\u4e2d\u5f80\u5f80\u88ab\u5ffd\u7565\u3002<\/p>\n<p>\u57fa\u4e8e\u8fd9\u4e00\u89c2\u5bdf\uff0c\u6211\u4eec\u63d0\u51fa\u4e86\u591a\u6a21\u6001\u63a8\u8350\u89e3\u51b3\u65b9\u6848\uff0c\u6838\u5fc3\u601d\u60f3\u662f\u5728\u4f20\u7edfID\u7279\u5f81\u57fa\u7840\u4e0a\u5f15\u5165\u5546\u6237\u7684\u56fe\u50cf\u548c\u6587\u672c\u4fe1\u606f\uff0c\u901a\u8fc7\u591a\u6a21\u6001\u8868\u5f81\u5b66\u4e60\u66f4\u597d\u5730\u5efa\u6a21\u5546\u6237\u95f4\u7684\u8bed\u4e49\u76f8\u4f3c\u6027\uff0c\u4ece\u800c\u63d0\u5347\u63a8\u8350\u7cfb\u7edf\u5bf9\u7528\u6237\u5174\u8da3\u7684\u7406\u89e3\u548c\u9884\u6d4b\u80fd\u529b\u3002<\/p>\n<h3><strong>3.1 \u7cfb\u7edf\u67b6\u6784\u8bbe\u8ba1<\/strong><\/h3>\n<p>\u9488\u5bf9\u672c\u5730\u751f\u6d3b\u63a8\u8350\u573a\u666f\u7684\u7279\u6b8a\u6027\uff0c\u6211\u4eec\u6784\u5efa\u4e86\u57fa\u4e8e\u53cc\u5854\u6a21\u578b\u8868\u5f81\u7684\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u3002\u8003\u8651\u5230\u9996\u9875\u5546\u6237\u63a8\u8350\u7684\u4e1a\u52a1\u7279\u70b9\uff0c\u6211\u4eec\u8bbe\u8ba1\u4e86\u5c42\u6b21\u5316\u7684\u5546\u6237\u5185\u5bb9\u8868\u5f81\u5efa\u6a21\u65b9\u6cd5\u3002<\/p>\n<h3>3.1.1 \u5546\u6237\u8868\u5f81\u5efa\u6a21\u7b56\u7565<\/h3>\n<p><strong>\u57fa\u4e8e\u641c\u7d22\u884c\u4e3a\u5efa\u6a21\u7684\u52a8\u673a**<\/strong>\uff1a** <\/p>\n<p>\u5728\u63a8\u8350\u573a\u666f\u4e2d\uff0c\u7528\u6237\u7684\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u94fe\u80fd\u591f\u51c6\u786e\u53cd\u6620\u7528\u6237\u7684\u771f\u5b9e\u5174\u8da3\u504f\u597d\u3002\u76f8\u6bd4\u4e8e\u4f20\u7edf\u7684\u57fa\u4e8e\u70b9\u51fb\u6216\u6d4f\u89c8\u884c\u4e3a\u7684\u5efa\u6a21\u65b9\u5f0f\uff0c\u641c\u7d22\u884c\u4e3a\u5305\u542b\u4e86\u7528\u6237\u7684\u660e\u786e\u610f\u56fe\u8868\u8fbe\uff0c\u800c\u6700\u7ec8\u7684\u8d2d\u4e70\u8f6c\u5316\u5219\u9a8c\u8bc1\u4e86\u8fd9\u79cd\u610f\u56fe\u7684\u771f\u5b9e\u6027\u3002\u56e0\u6b64\uff0c\u6211\u4eec\u9009\u62e9\u57fa\u4e8e\u7528\u6237\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u6a21\u5f0f\u6765\u5b9a\u4e49\u5e97\u94fa\u5185\u5bb9\u4e0e\u7528\u6237\u5174\u8da3\u7684\u8bed\u4e49\u76f8\u4f3c\u6027\u3002<\/p>\n<p><strong>\u7528\u6237\u884c\u4e3a\u6a21\u5f0f\u5206\u6790**<\/strong>\uff1a** <\/p>\n<p>\u7528\u6237\u7684\u641c\u7d22\u884c\u4e3a\u5448\u73b0\u591a\u6837\u5316\u7279\u70b9\uff1a\u65e2\u6709&quot;\u641c\u54c1\u627e\u5e97&quot;\uff08\u5982\u641c\u7d22&quot;\u9ebb\u8fa3\u70eb&quot;\u5bfb\u627e\u76f8\u5173\u5e97\u94fa\uff09\uff0c\u4e5f\u6709&quot;\u76f4\u63a5\u641c\u5e97&quot;\uff08\u5982\u641c\u7d22&quot;\u6d77\u5e95\u635e&quot;\uff09\uff0c\u8fd8\u6709\u590d\u5408\u641c\u7d22\u610f\u56fe\uff08\u5982&quot;\u5ddd\u83dc\u706b\u9505&quot;\uff09\u3002\u8fd9\u4e9b\u4e0d\u540c\u7684\u641c\u7d22\u6a21\u5f0f\u53cd\u6620\u4e86\u7528\u6237\u5bf9\u5e97\u94fa\u8ba4\u77e5\u7684\u591a\u4e2a\u7ef4\u5ea6\u3002<\/p>\n<p><strong>\u5e97\u94fa\u5185\u5bb9\u8868\u5f81\u8bbe\u8ba1**<\/strong>\uff1a**<\/p>\n<p>\u4e3a\u5339\u914d\u7528\u6237\u591a\u6837\u5316\u7684\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u6a21\u5f0f\uff0c\u6211\u4eec\u91c7\u7528\u4e86\u5e97\u94fa\u6574\u4f53\u8868\u5f81\u7684\u5efa\u6a21\u65b9\u6cd5\uff1a<\/p>\n<ul>\n<li>\u5e97\u94fa\u57fa\u7840\u4fe1\u606f\uff1a\u5e97\u94falogo\u56fe + \u5e97\u94fa\u63cf\u8ff0\u6587\u672c\uff0c\u4f53\u73b0\u5e97\u94fa\u54c1\u724c\u7279\u8272\u548c\u57fa\u672c\u4fe1\u606f\uff1b<\/li>\n<li>\u5546\u54c1\u7ec4\u5408\u4fe1\u606f\uff1a\u8be5\u5e97\u94fa\u70ed\u9500top10\u5546\u54c1\u7684\u56fe\u6587\u4fe1\u606f\uff08\u5546\u54c1\u56fe\u7247 + recaption\uff09\uff0c\u53cd\u6620\u5e97\u94fa\u7684\u7ecf\u8425\u7279\u8272\u548c\u5546\u54c1\u4f9b\u7ed9\u80fd\u529b\uff1b<\/li>\n<\/ul>\n<p>\u8fd9\u79cd\u8bbe\u8ba1\u80fd\u591f\u540c\u65f6\u6ee1\u8db3\u7528\u6237&quot;\u770b\u5e97\u94fa\u54c1\u724c&quot;\u548c&quot;\u770b\u5546\u54c1\u7279\u8272&quot;\u7684\u4e0d\u540c\u51b3\u7b56\u9700\u6c42\u3002<\/p>\n<h3>3.1.2 \u4e09\u9636\u6bb5\u8bad\u7ec3\u7b56\u7565<\/h3>\n<p>\u57fa\u4e8e\u5546\u6237\u8868\u5f81\u5efa\u6a21\u7684\u8bbe\u8ba1\u601d\u8def\uff0c\u6211\u4eec\u5236\u5b9a\u4e86\u4e09\u9636\u6bb5\u6e10\u8fdb\u5f0f\u8bad\u7ec3\u7b56\u7565\uff1a<\/p>\n<p><strong>\u9636\u6bb5\u4e00\uff1a\u9886\u57df\u9002\u914d\u9884\u8bad\u7ec3<\/strong><\/p>\n<p>\u5728\u672c\u5730\u751f\u6d3b\u56fe\u6587\u6570\u636e\u4e0a\u8fdb\u884c\u8de8\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60\uff0c\u5efa\u7acb\u9002\u914d\u5782\u76f4\u9886\u57df\u7684\u57fa\u7840\u8868\u5f81\u80fd\u529b\u3002\u8be5\u9636\u6bb5\u9700\u8981\u540c\u65f6\u8bad\u7ec3\u4e09\u7c7b\u56fe\u6587\u5bf9\u9f50\u4efb\u52a1\uff1a<\/p>\n<ul>\n<li>\u5546\u54c1\u7ea7\u56fe\u6587\u5bf9\u9f50\uff1a\u5efa\u7acb\u5355\u4e2a\u5546\u54c1\u7684\u56fe\u6587\u8868\u5f81\u57fa\u7840\uff1b<\/li>\n<li>\u5546\u6237\u7ea7\u56fe\u6587\u5bf9\u9f50\uff1a\u5b66\u4e60\u5546\u6237\u54c1\u724c\u4fe1\u606f\u7684\u56fe\u6587\u8868\u5f81\uff1b<\/li>\n<li>\u5546\u6237+\u5546\u54c1\u6df7\u5408\u5bf9\u9f50\uff1a\u5efa\u7acb\u5546\u6237\u6574\u4f53\u8868\u5f81\u7684\u56fe\u6587\u878d\u5408\u80fd\u529b\uff1b<\/li>\n<\/ul>\n<p><strong>\u9636\u6bb5\u4e8c\uff1a\u884c\u4e3a\u9a71\u52a8\u5fae\u8c03<\/strong><\/p>\n<p>\u5229\u7528\u771f\u5b9e\u7684\u7528\u6237\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u6570\u636e\u8fdb\u884c\u5bf9\u9f50\u8bad\u7ec3\u3002\u6839\u636e\u7528\u6237\u67e5\u8be2\u7684\u590d\u6742\u5ea6\u548c\u5e97\u94fa\u8868\u5f81\u7684\u4e30\u5bcc\u5ea6\uff0c\u8bad\u7ec3\u6570\u636e\u5305\u542b\u5355query-\u5355\u56fe\u548c\u591aquery-\u591a\u56fe\u4e24\u7c7b\uff0c\u901a\u8fc7\u6df7\u5408\u8bad\u7ec3\u786e\u4fdd\u6a21\u578b\u80fd\u591f\u5904\u7406\u4e0d\u540c\u590d\u6742\u5ea6\u7684\u5339\u914d\u573a\u666f\uff0c\u8be5\u9636\u6bb5\u540c\u6837\u5305\u542b\u4e09\u7c7b\u5bf9\u9f50\u4efb\u52a1\uff1a<\/p>\n<ul>\n<li>Query-Text\u5bf9\u9f50\uff1a\u7528\u6237\u67e5\u8be2\u4e0e\u5546\u6237\u6587\u672c\u63cf\u8ff0\u7684\u8bed\u4e49\u5339\u914d\uff1b<\/li>\n<li>Query-Image\u5bf9\u9f50\uff1a\u7528\u6237\u67e5\u8be2\u4e0e\u5546\u6237\u89c6\u89c9\u5185\u5bb9\u7684\u8bed\u4e49\u5bf9\u9f50\uff1b<\/li>\n<li>Query-Fusion\u5bf9\u9f50\uff1a\u7528\u6237\u67e5\u8be2\u4e0e\u5546\u6237\u6574\u4f53\u878d\u5408\u8868\u5f81\u7684\u7efc\u5408\u5bf9\u9f50\uff1b<\/li>\n<\/ul>\n<p>\u901a\u8fc7\u591a\u7ef4\u5ea6\u7684\u5bf9\u9f50\u5b66\u4e60\uff0c\u786e\u4fdd\u5546\u6237\u6574\u4f53\u8868\u5f81\u80fd\u591f\u5168\u9762\u6355\u6349\u7528\u6237\u7684\u771f\u5b9e\u8d2d\u4e70\u610f\u56fe\u3002<\/p>\n<p><strong>\u9636\u6bb5\u4e09\uff1a\u7279\u5f81\u878d\u5408\u4f18\u5316<\/strong><\/p>\n<p>\u53c2\u8003\u5de5\u4f5c[10]\uff0c\u901a\u8fc7SimTier\u548cMAKE\u6280\u672f\u5b9e\u73b0\u591a\u6a21\u6001\u8868\u5f81\u4e0e\u4f20\u7edfID\u7279\u5f81\u7684\u6709\u6548\u6574\u5408\u3002<\/p>\n<p>\u8fd9\u79cd\u5c42\u6b21\u5316\u7684\u5efa\u6a21\u65b9\u6cd5\u65e2\u4fdd\u8bc1\u4e86\u5546\u6237\u8868\u5f81\u7684\u5b8c\u6574\u6027\uff0c\u53c8\u901a\u8fc7\u6e10\u8fdb\u5f0f\u8bad\u7ec3\u7b56\u7565\u786e\u4fdd\u4e86\u4e0d\u540c\u7c92\u5ea6\u7279\u5f81\u7684\u6709\u6548\u5b66\u4e60\u548c\u878d\u5408\u3002\u8fd9\u79cd\u8bbe\u8ba1\u601d\u8def\u4ece\u7528\u6237\u771f\u5b9e\u884c\u4e3a\u51fa\u53d1\uff0c\u786e\u4fdd\u4e86\u5e97\u94fa\u8868\u5f81\u80fd\u591f\u6709\u6548\u6355\u6349\u7528\u6237\u5728\u63a8\u8350\u573a\u666f\u4e0b\u7684\u6f5c\u5728\u5174\u8da3\u3002<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p>\u6574\u4f53\u5efa\u6a21<\/p>\n<h3><strong>3.2 Pretrain\u5bf9\u9f50\u9886\u57df\u56fe\u6587<\/strong><\/h3>\n<h3>3.2.1 \u8bad\u7ec3\u6570\u636e\u6784\u5efa\u7b56\u7565<\/h3>\n<p>\u57fa\u4e8e\u672c\u5730\u751f\u6d3b\u4e1a\u52a1\u7279\u70b9\uff0c\u6211\u4eec\u8bbe\u8ba1\u4e86\u4e09\u7c7b\u56fe\u6587\u5bf9\u9f50\u6570\u636e\uff1a<\/p>\n<p><strong>\u5546\u54c1\u7ea7\u56fe\u6587\u5bf9**<\/strong>\uff1a**\u5546\u54c1\u56fe\u7247\u914d\u5408\u7c7b\u76ee\u3001\u5c5e\u6027\u4fe1\u606f\uff0c\u901a\u8fc7MLLM\u751f\u6210\u8be6\u7ec6\u7684recaption\uff0c\u7a81\u51fa\u83dc\u54c1\u7279\u8272\u3001\u53e3\u5473\u3001\u4ef7\u683c\u7b49\u5173\u952e\u4fe1\u606f\u3002\uff08MLLM\u6765\u81ea\u672c\u56e2\u961f\u8bad\u7ec3\u7684\u5782\u57df\u591a\u6a21\u6001\u5927\u6a21\u578b\uff09<\/p>\n<p><strong>\u5e97\u94fa\u7ea7\u56fe\u6587\u5bf9**<\/strong>\uff1a**\u5e97\u94fa\u5934\u56fe\u7ed3\u5408\u540d\u79f0\u3001\u7b80\u4ecb\u3001\u54c1\u7c7b\u4fe1\u606f\uff0c\u5229\u7528LLM\u8fdb\u884c\u63cf\u8ff0\u6269\u5145\uff0c\u5f3a\u8c03\u5e97\u94fa\u7279\u8272\u548c\u670d\u52a1\u4eae\u70b9\u3002<\/p>\n<p>\u6df7\u5408\u7ea7\u56fe\u6587\u5bf9\uff1a\u5e97\u94fa\u4e0e\u5546\u54c1\u591a\u56fe\u7ec4\u5408\uff0c\u6784\u9020\u53cd\u6620\u5b8c\u6574\u7ecf\u8425\u7279\u5f81\u7684\u7efc\u5408\u63cf\u8ff0\u3002<\/p>\n<h3>3.2.2 \u6a21\u578b\u67b6\u6784\u8bbe\u8ba1<\/h3>\n<p>\u6211\u4eec\u5728\u6a21\u578b\u67b6\u6784\u4e0a\u8bbe\u8ba1\u4e24\u79cd\u5efa\u6a21\u65b9\u6848\uff0c\u4e00\u662f\u91c7\u7528\u53cc\u5854\u5f0f\u7f16\u7801\u67b6\u6784\uff08EVL-sep\uff09\u5efa\u6a21\uff0c\u5e76\u901a\u8fc7\u8de8\u6a21\u6001\u6295\u5f71\u5c06\u6587\u672c\u4e0e\u89c6\u89c9\u8868\u5f81\u7edf\u4e00\u5728128\u7ef4\uff0c\u65e8\u5728\u63d0\u5347\u8ba1\u7b97\u6548\u7387\u7684\u540c\u65f6\uff0c\u5145\u5206\u5229\u7528SOTA\u7684\u89c6\u89c9\u4e0e\u6587\u672cencoder\u7684\u5404\u81ea\u4f18\u52bf\uff1b\u4e8c\u662f\u91c7\u7528\u8054\u5408\u8868\u5f81\uff08EVL-uni\uff09\u5efa\u6a21\uff0c\u5229\u7528MLLM\u5927\u6a21\u578b\u7edf\u4e00\u8868\u5f81\u80fd\u529b\uff0c\u57fa\u4e8e\u901a\u4e49\u5b9e\u9a8c\u5ba4\u63a8\u51fa\u7684GME\u591a\u6a21\u6001\u8868\u5f81\u6a21\u578b\u5b9e\u73b0\u56fe\u6587\u7edf\u4e00encode\u3002\u7531\u4e8e\u4e8c\u8005\u9664\u5efa\u6a21\u5916\u8bad\u7ec3\u601d\u8def\u57fa\u672c\u4e00\u81f4\uff0c\u672c\u6587\u91cd\u70b9\u9488\u5bf9\u6548\u679c\u66f4\u4f73\u53cc\u5854\u67b6\u6784\u7684EVL-sep\u8fdb\u884c\u4ecb\u7ecd\uff1a<\/p>\n<p><strong>\u89c6\u89c9\u7f16\u7801\u6a21\u5757\uff1a<\/strong> <\/p>\n<ul>\n<li>\u9aa8\u5e72\u7f51\u7edc\uff1aInternViT-300M-448px-V2.5\uff0c\u57fa\u4e8eInternViT-6B\u84b8\u998f\u5f97\u5230\uff1b<\/li>\n<li>\u8f93\u5165\u5904\u7406\uff1a\u52a8\u6001\u9ad8\u5206\u8fa8\u7387\u7b56\u7565\uff0c\u652f\u6301\u6700\u591a12\u4e2a448\u00d7448\u56fe\u5757+1\u4e2a\u5168\u5c40\u7f29\u7565\u56fe\uff1b<\/li>\n<li>\u7279\u5f81\u538b\u7f29\uff1aPixel Unshuffle\u91cd\u6392\u5217 + Pooling\u964d\u7ef4\uff0c\u6709\u6548\u51cf\u5c11\u89c6\u89c9token\u6570\u91cf\uff1b<\/li>\n<li>\u8de8\u6a21\u6001\u6295\u5f71\uff1a\u901a\u8fc74\u5c42MLP\u7f51\u7edc(LayerNorm\u2192Linear\u2192GELU\u2192Linear)\u5c06\u538b\u7f29\u540e\u7684\u89c6\u89c9\u7279\u5f81\u6620\u5c04\u5230128\u7ef4\u5171\u4eab\u8bed\u4e49\u7a7a\u95f4\uff1b<\/li>\n<\/ul>\n<p><strong>\u6587\u672c\u7f16\u7801\u6a21\u5757\uff1a<\/strong><\/p>\n<ul>\n<li>\u9aa8\u5e72\u7f51\u7edc\uff1aGTE-Qwen2-1.5B-instruct\uff0c\u4fdd\u6301decoder\u67b6\u6784\u7279\u6027\uff1b<\/li>\n<li>\u8868\u5f81\u63d0\u53d6\uff1a\u91c7\u7528last_token_pool\u7b56\u7565\u63d0\u53d6\u5e8f\u5217\u6700\u540e\u6709\u6548token\u7684\u7279\u5f81\uff0c\u81ea\u9002\u5e94\u5904\u7406\u4e0d\u540cpadding\u683c\u5f0f\uff1b<\/li>\n<li>\u8de8\u6a21\u6001\u6295\u5f71\uff1a\u901a\u8fc7\u7ebf\u6027\u53d8\u6362\u5c42(\u65e0bias)\u5c061.5B\u7ef4\u6587\u672c\u7279\u5f81\u76f4\u63a5\u6620\u5c04\u5230128\u7ef4\u5171\u4eab\u8bed\u4e49\u7a7a\u95f4\uff1b<\/li>\n<li>\u4f18\u5316\u7b56\u7565\uff1a\u5229\u7528LLM\u9884\u8bad\u7ec3\u77e5\u8bc6\uff0c\u589e\u5f3a\u6587\u672c\u8bed\u4e49\u7406\u89e3\u80fd\u529b\uff1b<\/li>\n<\/ul>\n<p><strong>\u7279\u5f81\u5bf9\u9f50\u673a\u5236\uff1a<\/strong><\/p>\n<ul>\n<li>\u5171\u4eab\u8bed\u4e49\u7a7a\u95f4\uff1a\u89c6\u89c9\u548c\u6587\u672c\u7279\u5f81\u5747\u6295\u5f71\u5230128\u7ef4\u5411\u91cf\u7a7a\u95f4\uff0c\u4fbf\u4e8e\u8ba1\u7b97\u8de8\u6a21\u6001\u76f8\u4f3c\u5ea6\uff1b<\/li>\n<li>L2\u5f52\u4e00\u5316\uff1a\u5bf9\u6240\u6709\u6a21\u6001\u7279\u5f81\u8fdb\u884c\u5355\u4f4d\u7403\u9762\u5f52\u4e00\u5316\uff0c\u786e\u4fdd\u4f59\u5f26\u76f8\u4f3c\u5ea6\u8ba1\u7b97\u7684\u7a33\u5b9a\u6027\uff1b<\/li>\n<li>\u6e29\u5ea6\u7f29\u653e\uff1a\u901a\u8fc7\u53ef\u5b66\u4e60\u7684logit_scale\u53c2\u6570(\u521d\u59cb\u503c2.6582)\u8c03\u8282\u76f8\u4f3c\u5ea6\u5206\u5e03\u9510\u5ea6\uff0c\u4f18\u5316\u5bf9\u6bd4\u5b66\u4e60\u6548\u679c\uff1b<\/li>\n<\/ul>\n<p>\u8fd9\u6837\u7684\u8bbe\u8ba1\u786e\u4fdd\u4e86\uff1a<\/p>\n<ol>\n<li><strong>\u7ef4\u5ea6\u7edf\u4e00**<\/strong>\uff1a**\u4e0d\u540c\u6a21\u6001\u7279\u5f81\u5728\u76f8\u540c\u7684128\u7ef4\u8bed\u4e49\u7a7a\u95f4\u4e2d\u8fdb\u884c\u6bd4\u8f83\uff1b<\/li>\n<li><strong>\u8ba1\u7b97\u6548\u7387**<\/strong>\uff1a**\u76f8\u6bd4\u539f\u59cb\u7279\u5f81\u7ef4\u5ea6\uff0c128\u7ef4\u5927\u5e45\u964d\u4f4e\u4e86\u76f8\u4f3c\u5ea6\u8ba1\u7b97\u7684\u590d\u6742\u5ea6\uff1b<\/li>\n<li><strong>\u8868\u5f81\u8d28\u91cf**<\/strong>\uff1a**\u901a\u8fc7\u4e13\u95e8\u7684\u6295\u5f71\u5c42\u5b66\u4e60\u6a21\u6001\u95f4\u7684\u8bed\u4e49\u5bf9\u5e94\u5173\u7cfb\uff1b<\/li>\n<\/ol>\n<p><strong>\u5bf9\u6bd4\u5b66\u4e60\u6846\u67b6**<\/strong>\uff1a** <\/p>\n<p>\u53cc\u5854\u7ed3\u6784\u901a\u8fc7InfoNCE\u635f\u5931\u8fdb\u884c\u8de8\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60\uff0c\u6b63\u6837\u672c\u4e3a\u56fe\u6587\u914d\u5bf9\uff0c\u8d1f\u6837\u672c\u91c7\u7528batch\u5185\u968f\u673a\u91c7\u6837\u3002InfoNCE\u635f\u5931\u51fd\u6570\u7684\u5b9a\u4e49\u4e0e\u793a\u4f8b\u8ba1\u7b97\u8fc7\u7a0b\u5982\u4e0b\uff1a<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143522807.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<ol>\n<li>n\u4e2a\u53e5\u5b50\u4e0en\u4e2a\u5bf9\u5e94\u7684\u56fe\u7247\u5206\u522b\u5d4c\u5165\u3001\u5206\u522b\u8fc7\u4e00\u4e2a\u6295\u5f71\u5c42\uff08\u5b66\u5355\u6a21\u8f6c\u591a\u6a21\uff0c\u5e76\u7edf\u4e00\u7ef4\u5ea6\uff09\u3001\u6b63\u5219\u5316\uff0c\u5f97\u5230\u4e24\u4e2a<code>n*d_e<\/code>\u7684\u5411\u91cfI\u548cT\uff1b<\/li>\n<li>\u4e8c\u8005\u70b9\u79ef\uff0c\u7ed3\u679c\u662f\u4e00\u4e2a\u5f62\u72b6\u4e3a <code>[n, n]<\/code> \u7684\u4f59\u5f26\u76f8\u4f3c\u5ea6\u77e9\u9635\uff0c\u5176\u4e2d\u6bcf\u4e2a\u5143\u7d20\u8868\u793a\u4e00\u4e2a\u56fe\u50cf\u4e0e\u4e00\u4e2a\u6587\u672c\u7684\u4f59\u5f26\u76f8\u4f3c\u5ea6\uff0c\u4f5c\u4e3alogits\u3002<\/li>\n<\/ol>\n<ul>\n<li>\u4e58\u4ee5 <code>np.exp(t)<\/code> \u662f\u901a\u8fc7\u6e29\u5ea6t\u5bf9\u76f8\u4f3c\u5ea6\u8fdb\u884c\u7f29\u653e\uff0c\u8bad\u7ec3\u4e2dt\u88ab\u8bbe\u7f6e\u4e3a\u53ef\u8bad\u7ec3\u53c2\u6570\uff0c\u56e0\u4e3a\u4f59\u5f26\u76f8\u4f3c\u5ea6\u8303\u56f40-1\uff0c\u4e0d\u592a\u9002\u5408\u76f4\u63a5\u4f5c\u4e3a\u4ea4\u53c9\u71b5\u7684logits\uff0c\u65e0\u4e0a\u9650\u533a\u5206\u5ea6\u597d\u4e00\u4e9b\uff1b<\/li>\n<li>logits\u4e0elabel\u7b97\u4ea4\u53c9\u71b5\u635f\u5931\uff08\u5bf9\u89d2\u7ebf\u5373\u6b63\u786e\u5339\u914d\u9879\u4e3alabel1\uff0c\u5176\u4ed6\u5373\u9519\u8bef\u5339\u914d\u9879\u4e3alabel0\uff09 \u3002<\/li>\n<\/ul>\n<h3>3.2.3 \u8bc4\u4f30<\/h3>\n<p><strong>\u8bc4\u4f30\u6307\u6807\u8bbe\u8ba1**<\/strong>\uff1a**<\/p>\n<p>\u91c7\u7528\u8de8\u6a21\u6001\u68c0\u7d22\u4efb\u52a1\u8bc4\u4f30\u6a21\u578b\u6027\u80fd\uff0c\u4f7f\u7528Recall@k\uff08k=1,5,10\uff09\u4f5c\u4e3a\u6838\u5fc3\u6307\u6807\u3002\u5177\u4f53\u65b9\u6cd5\u4e3a\uff1a\u5c06\u6d4b\u8bd5\u96c6\u56fe\u50cf\u548c\u6587\u672c\u5206\u522b\u7f16\u7801\u4e3a\u7279\u5f81\u5411\u91cf\uff0c\u901a\u8fc7\u4f59\u5f26\u76f8\u4f3c\u5ea6\u8ba1\u7b97\u5b9e\u73b0Text-to-Image\u548cImage-to-Text\u53cc\u5411\u68c0\u7d22\uff0c\u8bc4\u4f30\u6b63\u786e\u5339\u914d\u9879\u5728\u524dk\u4e2a\u7ed3\u679c\u4e2d\u7684\u53ec\u56de\u7387\u3002\u6d4b\u8bd5\u96c6\u6309\u5546\u54c1\u5355\u56fe\u3001\u5e97\u94fa\u5355\u56fe\u3001\u5e97\u94fa+\u5546\u54c1\u591a\u56fe\u4e09\u4e2a\u573a\u666f\u5206\u5c42\u91c7\u6837\uff0c\u786e\u4fdd\u4e0e\u8bad\u7ec3\u6570\u636e\u65e0\u91cd\u53e0\u3002<\/p>\n<p><strong>\u6b63\u8d1f\u6837\u672c\u6784\u9020\u7b56\u7565**<\/strong>\uff1a**<\/p>\n<p>\u6b63\u6837\u672c\u6784\u5efa\uff1a\u5546\u54c1\u56fe\u7247\u4e0erecaption\u63cf\u8ff0\u3001\u5546\u6237logo\u4e0e\u5546\u6237\u63cf\u8ff0\u3001\u5546\u6237\u6574\u4f53\u8868\u5f81\u4e0e\u7efc\u5408\u63cf\u8ff0\u5206\u522b\u6784\u6210\u4e09\u7c7b\u6b63\u6837\u672c\u5bf9\u3002<\/p>\n<p>\u8d1f\u6837\u672c\u91c7\u6837\uff1a\u91c7\u7528In-batch Negative Sampling\u7ed3\u5408\u56f0\u96be\u8d1f\u6837\u672c\u6316\u6398\u7b56\u7565\u3002\u56f0\u96be\u8d1f\u6837\u672c\u5305\u62ec\u540c\u7c7b\u76ee\u4e0d\u540c\u5546\u6237\u7684\u56fe\u6587\u5bf9\uff0c\u4ee5\u53ca\u57fa\u4e8e\u76f8\u4f3c\u5ea6\u8ba1\u7b97\u9009\u62e9\u7684\u9ad8\u76f8\u4f3c\u5ea6\u975e\u5339\u914d\u6837\u672c\uff0c\u63d0\u5347\u6a21\u578b\u7684\u7ec6\u7c92\u5ea6\u533a\u5206\u80fd\u529b\u3002<\/p>\n<p>\u6837\u672c\u8d28\u91cf\u63a7\u5236\uff1a\u901a\u8fc7\u89c4\u5219\u8fc7\u6ee4\u4f2a\u8d1f\u6837\u672c\uff0c\u79fb\u9664\u540c\u4e00\u5546\u6237\u7684\u4e0d\u540c\u8868\u8ff0\u7ec4\u5408\uff0c\u786e\u4fdd\u8d1f\u6837\u672c\u7684\u6709\u6548\u6027\u3002\u52a8\u6001\u8c03\u6574\u56f0\u96be\u8d1f\u6837\u672c\u6bd4\u4f8b\uff0c\u8bad\u7ec3\u540e\u671f\u9010\u6e10\u589e\u52a0\u56f0\u96be\u6837\u672c\u5360\u6bd4\u3002<\/p>\n<p><strong>\u8bc4\u4f30\u7ed3\u679c**<\/strong>\uff1a**<\/p>\n<p>\u7ecf\u8fc7\u9886\u57df\u9002\u914d\u9884\u8bad\u7ec3\u7684\u53cc\u5854\u5f0f\u6a21\u578b\u5728\u8de8\u6a21\u6001\u68c0\u7d22\u4efb\u52a1\u4e0a\u8fbe\u523090.8%\u7684\u6574\u4f53\u53ec\u56de\u7387\uff0c\u663e\u8457\u4f18\u4e8eSOTA\u901a\u7528\u6a21\u578bGME-qwen2-vl-7b\uff083584\u7ef4\uff09\u768466.5%\uff0c\u5728\u5546\u54c1\u5355\u9879\u7684\u56fe\u6587\u641c\u7d22\u4e0a\u8fbe\u523093.7%\u7684\u6574\u4f53\u53ec\u56de\u7387\uff0c\u663e\u8457\u4f18\u4e8e\u56e2\u961f\u8fc7\u5f80\u5728\u5546\u54c1\u56fe\u6587\u641c\u7d22\u573a\u666f\u8bad\u7ec3\u7684MS-ALBEF\uff08128\u7ef4\uff09\u768475.8%\u3002\u5728\u5355\u56fe\u5546\u54c1\u3001\u5355\u56fe\u5546\u6237\u3001\u591a\u56fe\u6df7\u5408\u573a\u666f\u4e0b\u5206\u522b\u8fbe\u523093.7%\u300182.0%\u300196.7%\u7684\u5e73\u5747\u53ec\u56de\u8868\u73b0\uff0c\u9a8c\u8bc1\u4e86\u9886\u57df\u9002\u914d\u8bad\u7ec3\u548c\u7cbe\u5fc3\u8bbe\u8ba1\u7684\u6837\u672c\u6784\u9020\u7b56\u7565\u7684\u6709\u6548\u6027\u3002<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E) <img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143522953.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>\u9884\u8bad\u7ec3\u6548\u679c\u5bf9\u6bd4-\u7efc\u5408\u6027\u80fd<\/p>\n<h3><strong>3.3 SFT\u5bf9\u9f50\u7528\u6237\u504f\u597d<\/strong><\/h3>\n<h3>3.3.1 \u7528\u6237\u504f\u597d\u5efa\u6a21\u601d\u8def<\/h3>\n<p>\u9884\u8bad\u7ec3\u9636\u6bb5\u867d\u7136\u5efa\u7acb\u4e86\u591a\u7c92\u5ea6\u7684\u56fe\u6587\u8868\u5f81\u57fa\u7840\uff0c\u4f46\u8fd9\u4e9b\u8868\u5f81\u4e3b\u8981\u57fa\u4e8e\u5185\u5bb9\u76f8\u4f3c\u6027\uff0c\u7f3a\u4e4f\u7528\u6237\u771f\u5b9e\u504f\u597d\u7684\u6307\u5bfc\u3002\u5728\u63a8\u8350\u573a\u666f\u4e2d\uff0c\u6211\u4eec\u9700\u8981\u7406\u89e3\u7528\u6237\u7684\u6f5c\u5728\u5174\u8da3\uff0c\u800c\u7528\u6237\u7684\u641c\u7d22-\u8d2d\u4e70\u884c\u4e3a\u94fe\u6070\u597d\u63d0\u4f9b\u4e86\u8fd9\u79cd\u5174\u8da3\u7684\u660e\u786e\u8868\u8fbe\u3002\u6211\u4eec\u7684\u6838\u5fc3\u7406\u5ff5\u53ef\u5f52\u7eb3\u5982\u4e0b\uff1a<\/p>\n<p><strong>1. \u884c\u4e3a\u5373\u5174\u8da3**<\/strong>\uff1a**\u7528\u6237\u7684\u641c\u7d22\u67e5\u8be2\u76f4\u63a5\u53cd\u6620\u5176\u9700\u6c42\u610f\u56fe\uff0c\u6700\u7ec8\u7684\u8d2d\u4e70\u8f6c\u5316\u9a8c\u8bc1\u4e86\u8fd9\u79cd\u610f\u56fe\u7684\u771f\u5b9e\u6027<\/p>\n<p><strong>2. \u8bed\u4e49\u6865\u6881**<\/strong>\uff1a**\u901a\u8fc7\u7528\u6237\u67e5\u8be2\u4f5c\u4e3a\u6865\u6881\uff0c\u53ef\u4ee5\u5efa\u7acb\u5e97\u94fa\u5185\u5bb9\u4e0e\u7528\u6237\u5174\u8da3\u4e4b\u95f4\u7684\u8bed\u4e49\u5173\u8054<\/p>\n<p><strong>3. \u63a8\u8350\u9002\u914d**<\/strong>\uff1a**\u5c06\u57fa\u4e8e\u641c\u7d22\u884c\u4e3a\u5b66\u5230\u7684\u8bed\u4e49\u76f8\u4f3c\u6027\u8fc1\u79fb\u5230\u63a8\u8350\u573a\u666f\uff0c\u63d0\u5347\u9996\u9875\u5e97\u94fa\u63a8\u8350\u7684\u7cbe\u51c6\u5ea6<\/p>\n<h3>3.3.2 \u8bad\u7ec3\u6570\u636e\u6784\u5efa<\/h3>\n<p><strong>\u6570\u636e\u6784\u5efa\u4e0e\u8d28\u91cf\u63a7\u5236\uff1a<\/strong><\/p>\n<p>\u57fa\u4e8e\u7528\u6237\u641c\u7d22\u540e\u4ea7\u751f\u8d2d\u4e70\u8f6c\u5316\u7684\u771f\u5b9e\u884c\u4e3a\u6570\u636e\uff0c\u6784\u5efa&lt;\u7528\u6237\u67e5\u8be2, \u8d2d\u4e70\u5e97\u94fa\u56fe\u7247, \u8d2d\u4e70\u5e97\u94fa\u63cf\u8ff0&gt;\u4e09\u5143\u7ec4\u4f5c\u4e3a\u6b63\u6837\u672c\u3002\u91c7\u7528batch\u5185\u5176\u4ed6\u5e97\u94fa\u7684\u56fe\u6587\u4fe1\u606f\u4f5c\u4e3a\u8d1f\u6837\u672c\uff0c\u540c\u65f6\u5f15\u5165\u56f0\u96be\u8d1f\u6837\u672c\u6316\u6398\u7b56\u7565\uff0c\u9009\u62e9\u4e0e\u67e5\u8be2\u76f8\u4f3c\u5ea6\u8f83\u9ad8\u4f46\u975e\u5339\u914d\u7684\u6837\u672c\u8fdb\u884c\u8bad\u7ec3\u3002\u4e3a\u786e\u4fdd\u6570\u636e\u4ee3\u8868\u6027\u548c\u8d28\u91cf\uff0c\u7edf\u8ba1\u5168\u91cf\u9910\u996e\u5e97\u94fa\u7684\u9ad8\u9891\u67e5\u8be2\u8bcd\uff0c\u9009\u62e9top10\u67e5\u8be2\u5bf9\u5e94\u7684top5\u6210\u4ea4\u5e97\u94fa\u6784\u5efa\u8bad\u7ec3\u96c6\u3002<\/p>\n<p><strong>\u6570\u636e\u7c7b\u578b\u4e0e\u8bad\u7ec3\u7b56\u7565\uff1a<\/strong><\/p>\n<p>\u6839\u636e\u67e5\u8be2\u590d\u6742\u5ea6\u548c\u5e97\u94fa\u8868\u5f81\u4e30\u5bcc\u5ea6\uff0c\u8bad\u7ec3\u6570\u636e\u5305\u542b\u4e24\u7c7b\uff1a<\/p>\n<ul>\n<li>\u5355query-\u5355\u56fe\u6570\u636e\uff1a\u7b80\u5355\u67e5\u8be2\u8bcd\u4e0e\u5e97\u94fa\u57fa\u7840\u56fe\u6587\u4fe1\u606f\u7684\u5339\u914d\uff1b<\/li>\n<li>\u591aquery-\u591a\u56fe\u6570\u636e\uff1a\u590d\u5408\u67e5\u8be2\u610f\u56fe\u4e0e\u5e97\u94fa\u6574\u4f53\u8868\u5f81\uff08\u5305\u542b\u5546\u54c1\u7ec4\u5408\u4fe1\u606f\uff09\u7684\u5339\u914d\uff1b<\/li>\n<\/ul>\n<p>\u5728\u5b9e\u9645\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\uff0c\u6211\u4eec\u5bf9\u6bd4\u4e86\u4e24\u79cd\u8bad\u7ec3\u7b56\u7565\uff1a\u5206\u9636\u6bb5\u8bad\u7ec3\uff08\u5148\u8bad\u7ec3\u5355\u56fe\u6570\u636e\uff0c\u518d\u8bad\u7ec3\u591a\u56fe\u6570\u636e\uff09\u548c\u6df7\u5408\u8bad\u7ec3\uff08\u540c\u65f6\u8bad\u7ec3\u4e24\u7c7b\u6570\u636e\uff09\u3002\u5b9e\u9a8c\u53d1\u73b0\uff0c\u5206\u9636\u6bb5\u8bad\u7ec3\u4f1a\u5bfc\u81f4\u6a21\u578b\u5728\u5355\u56fe\u5339\u914d\u4efb\u52a1\u4e0a\u7684\u6027\u80fd\u663e\u8457\u4e0b\u964d\uff0c\u8fd9\u662f\u7531\u4e8e\u540e\u671f\u591a\u56fe\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u6a21\u578b\u5bf9\u65e9\u671f\u5b66\u4e60\u7684\u5355\u56fe\u5339\u914d\u80fd\u529b\u4ea7\u751f\u4e86\u9057\u5fd8\u3002\u4e3a\u4e86\u907f\u514d\u8fd9\u79cd\u707e\u96be\u6027\u9057\u5fd8\u95ee\u9898\uff0c\u6211\u4eec\u6700\u7ec8\u91c7\u7528\u4e86\u6df7\u5408\u8bad\u7ec3\u7b56\u7565\uff0c\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u540c\u65f6\u4f7f\u7528\u5355query-\u5355\u56fe\u548c\u591aquery-\u591a\u56fe\u4e24\u7c7b\u6570\u636e\uff0c\u786e\u4fdd\u6a21\u578b\u5728\u4e0d\u540c\u590d\u6742\u5ea6\u4efb\u52a1\u4e0a\u7684\u5747\u8861\u8868\u73b0\u3002<\/p>\n<h3>3.3.3 \u7279\u5f81\u878d\u5408\u4e0e\u635f\u5931\u8bbe\u8ba1<\/h3>\n<p><strong>\u591a\u6a21\u6001\u7279\u5f81\u878d\u5408\uff1a<\/strong><\/p>\n<p>\u5728\u53cc\u5854\u5f0f\u67b6\u6784\u57fa\u7840\u4e0a\uff0c\u6211\u4eec\u589e\u52a0\u4e86Fusion Module\u4ee5\u5b9e\u73b0\u591a\u6a21\u7279\u5f81\u878d\u5408\uff0c\u91c7\u7528cross-attention\u673a\u5236\u4f5c\u4e3a\u591a\u6a21\u6001\u7f16\u7801\u5668\uff0c\u63a5\u6536\u56fe\u50cf\u7279\u5f81\u548c\u6587\u672c\u7279\u5f81\u4f5c\u4e3a\u8f93\u5165\uff0c\u8f93\u51fa\u878d\u5408\u540e\u7684128\u7ef4\u591a\u6a21\u6001\u8868\u5f81\u3002\u5177\u4f53\u800c\u8a00\uff0c\u6211\u4eec\u5bf9\u6bd4\u4e86\u4e0d\u540c\u5c3a\u5bf8MLPFusion\u3001TfnFusion\u3001LmfFusion\u3001T5Fusion\u7b49\u4e0d\u540c\u878d\u5408\u65b9\u6848\uff0c\u6700\u7ec8\u53c2\u8003\u5de5\u4f5c[21]\uff0c\u4f7f\u7528\u6548\u679c\u6700\u4f73\u7684T5 Feature Fusion\u6a21\u5757\u4ee5\u66f4\u6709\u6548\u7684\u5efa\u6a21\u8de8\u6a21\u6001\u4ea4\u4e92\u4fe1\u606f\u3002<\/p>\n<p><strong>\u591a\u8def\u5bf9\u9f50\u8bbe\u8ba1\uff1a<\/strong><\/p>\n<p>\u5355\u4e00\u7684\u56fe\u6587\u5bf9\u9f50\u65b9\u6cd5\u5bf9\u4e0d\u540c\u6a21\u6001\u4fe1\u606f\u7684\u5229\u7528\u4e0d\u591f\u5145\u5206\u3002\u867d\u7136\u5728\u591a\u6a21\u6001\u878d\u5408\u65f6\u540c\u65f6\u4f7f\u7528\u4e86\u56fe\u50cf\u3001\u6587\u672c\u4fe1\u606f\uff0c\u4f46\u65e0\u6cd5\u4fdd\u8bc1\u8f93\u51fa\u7684\u591a\u6a21\u6001\u7279\u5f81\u80fd\u591f\u5e73\u8861\u5229\u7528\u4e0d\u540c\u6a21\u6001\u3002\u53ef\u80fd\u51fa\u73b0\u67e5\u8be2\u4e0e\u5355\u4e00\u6a21\u6001\u5bf9\u9f50\u826f\u597d\uff0c\u4f46\u4e0e\u878d\u5408\u7279\u5f81\u8ddd\u79bb\u8f83\u8fdc\u7684\u60c5\u51b5\uff0c\u5bfc\u81f4\u591a\u6a21\u6001\u4fe1\u606f\u6ca1\u6709\u5f88\u597d\u5730\u534f\u540c\u5de5\u4f5c\u3002<\/p>\n<p>\u4e3a\u4f7f\u6a21\u578b\u5b66\u5230\u7684\u8868\u5f81\u80fd\u591f\u5145\u5206\u5229\u7528\u56fe\u6587\u4fe1\u606f\uff0c\u6211\u4eec\u5728\u5fae\u8c03\u9636\u6bb5\u5c06\u9884\u8bad\u7ec3\u65f6\u7684\u56fe\u6587\u5bf9\u9f50\u4efb\u52a1\u6269\u5c55\u4e3a\u591a\u6a21\u6001\u5bf9\u9f50\u4efb\u52a1\uff0c\u8bbe\u8ba1\u4e86\u4e09\u7c7b\u5bf9\u6bd4\u5b66\u4e60\u635f\u5931\uff1a<\/p>\n<ul>\n<li><strong>Query-Image\u5bf9\u9f50**<\/strong>\uff1a**\u7ea6\u675f\u67e5\u8be2\u8868\u5f81\u4e0e\u56fe\u50cf\u8868\u5f81\u7684\u8bed\u4e49\u4e00\u81f4\u6027\uff0c\u786e\u4fdd\u89c6\u89c9\u4fe1\u606f\u7684\u6709\u6548\u5229\u7528\uff1b<\/li>\n<li><strong>Query-Text\u5bf9\u9f50**<\/strong>\uff1a**\u4fdd\u8bc1\u67e5\u8be2\u4e0e\u6587\u672c\u63cf\u8ff0\u7684\u8bed\u4e49\u5339\u914d\uff0c\u5f3a\u5316\u6587\u672c\u8bed\u4e49\u7684\u4f5c\u7528\uff1b<\/li>\n<li><strong>Query-Fusion\u5bf9\u9f50**<\/strong>\uff1a**\u5c06\u67e5\u8be2\u4e0e\u878d\u5408\u540e\u7684\u591a\u6a21\u6001\u8868\u5f81\u8fdb\u884c\u5bf9\u9f50\uff0c\u786e\u4fdd\u878d\u5408\u7279\u5f81\u7684\u6709\u6548\u6027\uff1b<\/li>\n<\/ul>\n<p><strong>\u635f\u5931\u51fd\u6570\u5b9e\u73b0\uff1a<\/strong><\/p>\n<p>\u6bcf\u7c7b\u5bf9\u9f50\u90fd\u91c7\u7528\u5bf9\u5076\u635f\u5931\u8bbe\u8ba1\uff0c\u5373\u540c\u65f6\u8ba1\u7b97\u6a21\u6001A\u5230\u6a21\u6001B\u548c\u6a21\u6001B\u5230\u6a21\u6001A\u7684\u5bf9\u6bd4\u635f\u5931\u3002\u901a\u8fc7\u6e29\u5ea6\u53c2\u6570\u03c4\u5bf9\u76f8\u4f3c\u5ea6\u8fdb\u884c\u7f29\u653e\uff0c\u4f7f\u7528softmax\u5f52\u4e00\u5316\u540e\u8ba1\u7b97\u4ea4\u53c9\u71b5\u635f\u5931\u3002\u6700\u7ec8\u7684\u591a\u6a21\u6001\u76d1\u7763\u635f\u5931\u4e3a\u5404\u4e2a\u5bf9\u6bd4\u635f\u5931\u7684\u52a0\u6743\u7ec4\u5408\uff1a <code>Loss=w1*q2t loss + w2*q2i loss + w3*q2f loss<\/code><\/p>\n<p>\u5728loss\u6743\u91cd\u5206\u914d\u4e0a\uff0c\u6d88\u878d\u5bf9\u6bd4\u53d1\u73b02:2:6-&gt;3:3:4\u5e26\u6765\u6574\u4f53\u53ec\u56de\u7387\u75310.856-&gt;0.860\u7684\u63d0\u5347\uff0c\u8bc1\u660e\u5404\u7c7bloss\u5728\u672c\u573a\u666f\u4efb\u52a1\u4e2d\u76f8\u5f53\u7684\u91cd\u8981\u6027\u3002<\/p>\n<p>\u901a\u8fc7\u591a\u6a21\u6001\u5bf9\u9f50\uff0c\u4ece\u4e0d\u540c\u89c6\u89d2\u6d88\u9664query\u3001\u56fe\u50cf\u3001\u6587\u672c\u3001\u878d\u5408\u7279\u5f81\u4e4b\u95f4\u7684\u8868\u5f81\u5dee\u5f02\u3002\u591a\u79cd\u635f\u5931\u7684\u53e0\u52a0\u4f7f\u5f97\u5404\u4e2a\u6a21\u6001\u8868\u5f81\u80fd\u591f\u6709\u6548\u5730\u5efa\u6a21\u5230\u540c\u4e00\u7279\u5f81\u7a7a\u95f4\uff0c\u5e76\u4e14\u4f7f\u5f97\u56fe\u50cf\u548c\u6587\u672c\u4e2d\u7684\u5173\u952e\u4fe1\u606f\u80fd\u591f\u4e3b\u5bfc\u5176\u591a\u6a21\u6001\u8868\u5f81\u3002<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143523722.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>SFT\u5efa\u6a21<\/p>\n<h3>3.3.4 \u8bc4\u4f30<\/h3>\n<p>\u4e3a\u9a8c\u8bc1\u6a21\u578b\u5728\u771f\u5b9e\u4e1a\u52a1\u573a\u666f\u4e0b\u7684\u8868\u73b0\uff0c\u6211\u4eec\u8bbe\u8ba1\u4e86\u57fa\u4e8e\u641c\u7d22\u8ba2\u5355\u573a\u666f\u7684\u8bc4\u4f30\u6846\u67b6\u3002\u8be5\u8bc4\u4f30\u65b9\u6cd5\u901a\u8fc7\u8ba1\u7b97\u7528\u6237\u67e5\u8be2\u4e0e\u5019\u9009\u5e97\u94fa\/\u5546\u54c1\u7684\u591a\u6a21\u6001\u76f8\u4f3c\u5ea6\u8fdb\u884c\u6392\u5e8f\uff0c\u9a8c\u8bc1\u7528\u6237\u771f\u5b9e\u4e0b\u5355\u9879\u5728\u76f8\u4f3c\u5ea6\u6392\u5e8f\u7ed3\u679c\u4e2d\u7684\u4f4d\u7f6e\u3002<\/p>\n<p>\u5177\u4f53\u800c\u8a00\uff0c\u6211\u4eec\u91c7\u7528Order Hitrate@k\u4f5c\u4e3a\u6838\u5fc3\u8bc4\u4f30\u6307\u6807\uff0c\u5176\u4e2dk\u53d6\u503c\u4e3a1\/5\/10\/20\/50\uff0c\u8861\u91cf\u7528\u6237\u771f\u5b9e\u4e0b\u5355\u9879\u5728\u6309\u76f8\u4f3c\u5ea6\u6392\u5e8f\u7684\u524dk\u4e2a\u7ed3\u679c\u4e2d\u7684\u547d\u4e2d\u60c5\u51b5\u3002<\/p>\n<p><strong>\u6d4b\u8bd5\u6837\u672c\u6784\u5efa\u7b56\u7565\uff1a<\/strong><\/p>\n<ul>\n<li><strong>\u6b63\u6837\u672c\u5b9a\u4e49**<\/strong>\uff1a**\u9009\u53d6\u4ea7\u751f\u5b9e\u9645\u4e0b\u5355\u884c\u4e3a\u7684\u7528\u6237\u641c\u7d22\u8bf7\u6c42\uff0c\u5c06\u5bf9\u5e94\u7684\u4e0b\u5355\u5e97\u94fa\u6216\u5546\u54c1\u6807\u8bb0\u4e3a\u76ee\u6807\u547d\u4e2d\u9879\uff1b<\/li>\n<li><strong>\u5019\u9009\u96c6\u6784\u9020**<\/strong>\uff1a**\u7ed3\u5408\u5730\u7406\u4f4d\u7f6e\u7ea6\u675f\u548c\u771f\u5b9e\u4e0b\u5355\u8bb0\u5f55\u6784\u5efa\u5019\u9009\u6c60\u3002\u5730\u7406\u5339\u914d\u9879\u901a\u8fc7geohash4\u7f16\u7801\u7b5b\u9009\u7528\u6237\u6240\u5728\u533a\u57df\u7684\u5e97\u94fa\uff0c\u968f\u673a\u91c7\u68371000\u4e2a\u4f5c\u4e3a\u5e72\u6270\u9879\uff1b\u540c\u65f6\u52a0\u5165\u7528\u6237\u5b9e\u9645\u4e0b\u5355\u7684\u5e97\u94fa\/\u5546\u54c1\u786e\u4fdd\u6b63\u6837\u672c\u5b58\u5728\uff1b<\/li>\n<li><strong>\u76f8\u4f3c\u5ea6\u8ba1\u7b97**<\/strong>\uff1a**\u4f7f\u7528\u8bad\u7ec3\u597d\u7684\u591a\u6a21\u6001\u6a21\u578b\u5206\u522b\u7f16\u7801\u7528\u6237\u67e5\u8be2\u548c\u5019\u9009\u5e97\u94fa\u7684\u56fe\u6587\u4fe1\u606f\uff0c\u901a\u8fc7\u4f59\u5f26\u76f8\u4f3c\u5ea6\u8ba1\u7b97\u8fdb\u884c\u6392\u5e8f\uff1b<\/li>\n<\/ul>\n<p><strong>\u8bc4\u4f30\u7ef4\u5ea6\u5212\u5206\uff1a<\/strong><\/p>\n<p>\u6309\u7167\u7528\u6237\u67e5\u8be2\u610f\u56fe\u5c06\u6d4b\u8bd5\u96c6\u5212\u5206\u4e3a\u4e09\u4e2a\u5b50\u96c6\uff1a\u5546\u54c1\u540d\/\u5546\u54c1\u54c1\u724c\u67e5\u8be2\u3001\u5546\u6237\u54c1\u724c\u67e5\u8be2\u3001\u5e97\u54c1\u6df7\u5408\u610f\u56fe\u67e5\u8be2\uff0c\u5206\u522b\u8bc4\u4f30\u6a21\u578b\u5728\u4e0d\u540c\u641c\u7d22\u573a\u666f\u4e0b\u7684\u8868\u73b0\u3002<\/p>\n<p><strong>\u8bc4\u4f30\u7ed3\u679c\uff1a<\/strong><\/p>\n<p>\u5728Order Hitrate@k\u8bc4\u4f30\u4e2d\uff0c\u5fae\u8c03\u540e\u7684EVL-sep\u6a21\u578b\u5728\u591a\u4e2a\u57fa\u7ebf\u6a21\u578b\u5bf9\u6bd4\u4e2d\u8868\u73b0\u4f18\u5f02\uff0c\u6574\u4f53\u5e73\u5747\u547d\u4e2d\u7387\u8fbe\u523086.0%\uff0c\u4f4d\u5c45\u6240\u6709\u6d4b\u8bd5\u6a21\u578b\u4e4b\u9996\u3002\u503c\u5f97\u6ce8\u610f\u7684\u662f\uff0cEVL\u6a21\u578b\u4ec5\u4f7f\u7528128\u7ef4\u8868\u5f81\u5411\u91cf\uff0c\u5374\u663e\u8457\u4f18\u4e8e\u4f7f\u75281536\u7ef4\u30013584\u7ef4\u8868\u5f81\u7684GME-Qwen2-VL\u7cfb\u5217\u6a21\u578b\u3002<\/p>\n<p>\u5404\u67e5\u8be2\u573a\u666f\u8be6\u7ec6\u5bf9\u6bd4\uff1a<\/p>\n<ul>\n<li><strong>\u5546\u54c1\u540d\/\u5546\u54c1\u54c1\u724c\u67e5\u8be2**<\/strong>\uff1a**EVL-sep\u5728HR@1\u4e0a\u8fbe\u523058.2%\uff0c\u76f8\u6bd4GME-Qwen2-VL-7B\uff0855.5%\uff09\u548cGME-Qwen2-VL-2B\uff0852.0%\uff09\u5206\u522b\u63d0\u53474.9%\u548c11.9%\uff1b\u5728HR@50\u4e0a\u8fbe\u523096.5%\uff0c\u8d85\u8d8a\u4e24\u4e2a\u57fa\u7ebf\u6a21\u578b\u768493.9%\u548c92.8%\u3002\u76f8\u6bd4EVL-uni\uff0860.0%\uff09\uff0cEVL-sep\u5728HR@1\u4e0a\u7565\u4f4e2.8%\uff0c\u4f46\u5728\u9ad8k\u503c\u573a\u666f\u4e0b\u8868\u73b0\u66f4\u7a33\u5b9a\u3002<\/li>\n<li><strong>\u5546\u6237\u54c1\u724c\u67e5\u8be2**<\/strong>\uff1a**EVL-sep\u8868\u73b0\u6700\u4e3a\u7a33\u5b9a\uff0cHR@1\u4e3a68.4%\uff0c\u867d\u7565\u4f4e\u4e8eGME-Qwen2-VL-7B\uff0871.9%\uff09\u548cGME-Qwen2-VL-2B\uff0870.7%\uff09\uff0c\u4f46\u5728HR@50\u4e0a\u8fbe\u523097.3%\uff0c\u4e0e\u57fa\u7ebf\u6a21\u578b\u6301\u5e73\u3002\u76f8\u6bd4EVL-uni\uff0868.1%\uff09\uff0cEVL-sep\u5728\u8be5\u573a\u666f\u4e0b\u8868\u73b0\u57fa\u672c\u4e00\u81f4\u3002<\/li>\n<li><strong>\u5e97\u54c1\u6df7\u5408\u610f\u56fe\u67e5\u8be2**<\/strong>\uff1a**\u4f5c\u4e3a\u6700\u5177\u6311\u6218\u6027\u7684\u573a\u666f\uff0cEVL-sep\u8868\u73b0\u7a81\u51fa\uff0cHR@1\u8fbe\u523053.2%\uff0c\u663e\u8457\u4f18\u4e8eGME-Qwen2-VL-7B\uff0857.7%\uff09\u548cGME-Qwen2-VL-2B\uff0854.5%\uff09\uff1b\u5728HR@50\u4e0a\u8fbe\u523098.6%\uff0c\u8d85\u8d8a\u6240\u6709\u57fa\u7ebf\u6a21\u578b\u3002\u76f8\u6bd4EVL-uni\uff0848.7%\uff09\uff0cEVL-sep\u5728HR@1\u4e0a\u9886\u51489.2%\uff0c\u4f53\u73b0\u4e86\u5206\u79bb\u5f0f\u67b6\u6784\u5728\u5904\u7406\u590d\u6742\u67e5\u8be2\u610f\u56fe\u4e0a\u7684\u4f18\u52bf\u3002<\/li>\n<\/ul>\n<p>\u7efc\u5408\u5206\u6790\u663e\u793a\uff0cEVL-sep\u6a21\u578b\u901a\u8fc7\u7cbe\u5fc3\u8bbe\u8ba1\u7684\u591a\u6a21\u6001\u878d\u5408\u67b6\u6784\u548c\u8bad\u7ec3\u7b56\u7565\uff0c\u5728\u4fdd\u8bc1\u6548\u679c\u7684\u540c\u65f6\u5927\u5e45\u964d\u4f4e\u4e86\u5b58\u50a8\u548c\u8ba1\u7b97\u6210\u672c\uff0c\u7279\u522b\u662f\u5728\u6df7\u5408\u610f\u56fe\u67e5\u8be2\u8fd9\u4e00\u6838\u5fc3\u4e1a\u52a1\u573a\u666f\u4e0b\u5c55\u73b0\u51fa\u663e\u8457\u4f18\u52bf\uff0c\u9a8c\u8bc1\u4e86\u6211\u4eec\u6280\u672f\u65b9\u6848\u7684\u6709\u6548\u6027\u3002<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143524035.png\" alt=\"\u56fe\u7247\" \/> <\/p>\n<p>SFT\u5404\u4efb\u52a1\u6548\u679c\u5bf9\u6bd4<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143524096.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>SFT\u7efc\u5408\u6548\u679c\u5bf9\u6bd4<\/p>\n<p><strong>3.4 Cotrain\u7279\u5f81\u878d\u5408\u4f18\u5316<\/strong><\/p>\n<p><strong>\u591a\u6a21\u6001\u4e0eID\u7279\u5f81\u878d\u5408\u7b56\u7565<\/strong><\/p>\n<p>\u591a\u6a21\u6001\u8868\u5f81\u4e0e\u4f20\u7edfID\u7279\u5f81\u7684\u878d\u5408\u9762\u4e34\u6839\u672c\u6027\u6311\u6218\uff1a\u57fa\u4e8e\u7528\u6237\u884c\u4e3a\u7684\u534f\u540c\u8fc7\u6ee4\u4e0e\u57fa\u4e8e\u5185\u5bb9\u7684\u591a\u6a21\u6001\u8868\u5f81\u5c5e\u4e8e\u4e0d\u540c\u7ef4\u5ea6\uff0c\u5b66\u4e60\u65b9\u5411\u7684\u4e00\u81f4\u6027\u8f83\u5dee\u3002\u76f4\u63a5\u5c06\u591a\u6a21\u6001\u7279\u5f81concat\u5230\u63a8\u8350\u6a21\u578b\u4e2d\u6548\u679c\u6709\u9650\uff0c\u9700\u8981\u66f4\u6709\u6548\u7684\u6574\u5408\u65b9\u5f0f\u3002<\/p>\n<p>\u6211\u4eec\u53c2\u8003\u5de5\u4f5c[10]\uff0c\u91c7\u7528\u4e86SimTier+MAKE\u6280\u672f\u89e3\u51b3\u8fd9\u4e00\u95ee\u9898\u3002SimTier\u901a\u8fc7\u76f8\u4f3c\u5ea6\u5206\u6876\u7684\u65b9\u5f0f\u5b9e\u73b0\u591a\u6a21\u6001\u4fe1\u606f\u7684&quot;ID\u5316&quot;\uff1a\u9996\u5148\u8ba1\u7b97\u76ee\u6807\u7269\u54c1\u4e0e\u7528\u6237\u5386\u53f2\u4ea4\u4e92\u7269\u54c1\u7684\u591a\u6a21\u6001\u8868\u5f81\u76f8\u4f3c\u5ea6\uff0c\u7136\u540e\u5c06\u76f8\u4f3c\u5ea6\u533a\u95f4\u5212\u5206\u4e3aN\u4e2a\u6876\uff0c\u7edf\u8ba1\u5404\u6876\u5185\u7684\u76f8\u4f3c\u5ea6\u6570\u91cf\u5f62\u6210N\u7ef4\u7279\u5f81\u5411\u91cf\uff0c\u6700\u540e\u4e0e\u539f\u6709ID\u7279\u5f81\u62fc\u63a5\u8f93\u5165\u63a8\u8350\u6a21\u578b\u3002\u8fd9\u79cd\u65b9\u6cd5\u901a\u8fc7&quot;ID\u5316&quot;\u964d\u4f4e\u4e86\u7279\u5f81\u7ef4\u5ea6\uff0c\u63d0\u5347\u4e86\u591a\u6a21\u6001\u4fe1\u606f\u4e0e\u63a8\u8350\u6a21\u578b\u7684\u517c\u5bb9\u6027\u3002<\/p>\n<p>MAKE\u6280\u672f\u5219\u89e3\u51b3\u4e86\u591a\u6a21\u6001\u7279\u5f81\u4e0eID\u7279\u5f81\u8bad\u7ec3\u8282\u594f\u4e0d\u4e00\u81f4\u7684\u95ee\u9898\u3002ID\u7279\u5f81\u6a21\u578b\u5b58\u5728&quot;one-epoch\u73b0\u8c61&quot;\uff08\u591a\u8f6e\u8bad\u7ec3\u5bfc\u81f4\u8fc7\u62df\u5408\uff09\uff0c\u800c\u591a\u6a21\u6001\u4fe1\u606f\u9700\u8981\u591a\u8f6e\u8bad\u7ec3\u624d\u80fd\u5145\u5206\u5b66\u4e60\u3002MAKE\u901a\u8fc7\u53c2\u6570\u5206\u79bb\u4f18\u5316\uff0c\u5c06\u591a\u6a21\u6001\u8868\u5f81\u53c2\u6570\u4e0eID\u7279\u5f81\u53c2\u6570\u72ec\u7acb\u5904\u7406\uff0c\u5141\u8bb8\u591a\u6a21\u6001\u90e8\u5206\u591a\u8f6e\u8bad\u7ec3\u800cID\u90e8\u5206\u4fdd\u6301\u5355\u8f6e\u8bad\u7ec3\uff0c\u540c\u65f6\u91c7\u7528\u6df1\u5ea6\u5174\u8da3\u7f51\u7edc\u5904\u7406\u7528\u6237\u884c\u4e3a\u5e8f\u5217\uff0c\u5b9e\u73b0\u591a\u6a21\u6001\u8868\u5f81\u4e0eID\u7279\u5f81\u7684\u6709\u6548\u878d\u5408\u3002<\/p>\n<p>![\u56fe\u7247](data:image\/svg+xml,%3C%3Fxml version='1.0' encoding='UTF-8'%3F%3E%3Csvg width='1px' height='1px' viewBox='0 0 1 1' version='1.1' xmlns='<a href=\"http:\/\/www.w3.org\/2000\/svg\">http:\/\/www.w3.org\/2000\/svg<\/a>' xmlns:xlink='<a href=\"http:\/\/www.w3.org\/1999\/xlink&#039;%3E%3Ctitle%3E%3C\/title%3E%3Cg\">http:\/\/www.w3.org\/1999\/xlink'%3E%3Ctitle%3E%3C\/title%3E%3Cg<\/a> stroke='none' stroke-width='1' fill='none' fill-rule='evenodd' fill-opacity='0'%3E%3Cg transform='translate(-249.000000, -126.000000)' fill='%23FFFFFF'%3E%3Crect x='249' y='126' width='1' height='1'%3E%3C\/rect%3E%3C\/g%3E%3C\/g%3E%3C\/svg%3E)<\/p>\n<p>SimTier+MAKE<\/p>\n<p><strong>\u8bc4\u4f30<\/strong><\/p>\n<p>\u7ecf\u8fc7\u7b2c\u4e09\u9636\u6bb5\u7279\u5f81\u878d\u5408\u4f18\u5316\u7684\u8054\u5408\u8bad\u7ec3\uff0c\u4e3a\u63a8\u8350\u7cfb\u7edf\u79bb\u7ebfAUC\u6307\u6807\u5e26\u6765\u663e\u8457\u63d0\u5347\uff1a<strong>\u5168\u91cf\u5546\u6237CTCVR AUC +0.19%\uff0cCTR AUC +0.17%<\/strong>\u3002<\/p>\n<p><strong>3.5 \u7f16\u7801\u5b9e\u8df5\u7ecf\u9a8c\u603b\u7ed3<\/strong><\/p>\n<h3>3.5.1 \u5efa\u6a21<\/h3>\n<p><strong>\u7075\u6d3b\u521d\u59cb\u5316\u7b56\u7565\u4e0e\u53c2\u6570\u9a8c\u8bc1<\/strong><\/p>\n<p>\u652f\u6301\u591a\u79cd\u521d\u59cb\u5316\u65b9\u5f0f\uff0c\u5305\u62ec\u914d\u7f6e+\u6a21\u578b\u7ec4\u5408\u7684\u76f4\u63a5\u521d\u59cb\u5316\u65b9\u6cd5\u548c\u5206\u79bb\u9884\u8bad\u7ec3\u6a21\u578b\u52a0\u8f7d\u7684\u521d\u59cb\u5316\u65b9\u6cd5\u3002\u53c2\u8003transformers vision_text_dual_encoder\u5b9e\u73b0from_vision_text_pretrained\u65b9\u6cd5\uff0c\u901a\u8fc7from_vision_text_configs()\u5b9e\u73b0\u53cc\u5854\u6a21\u578b\u914d\u7f6e\u81ea\u52a8\u878d\u5408\uff0c\u652f\u6301\u4e0d\u540c\u7c7b\u578b\u7684\u89c6\u89c9\u4e0e\u6587\u672c\u7f16\u7801\u5668\u3002\u5728\u6a21\u578b\u521d\u59cb\u5316\u540e\u901a\u8fc7torch.allclose()\u9a8c\u8bc1\u53c2\u6570\u4e00\u81f4\u6027\uff0c\u786e\u4fdd\u6743\u91cd\u6b63\u786e\u52a0\u8f7d\u3002<\/p>\n<p><strong>\u4ef7\u503c**<\/strong>\uff1a**\u63d0\u4f9b\u7075\u6d3b\u7684\u6a21\u578b\u6784\u5efa\u65b9\u5f0f\uff0c\u652f\u6301\u4ece\u5206\u79bb\u7684\u9884\u8bad\u7ec3\u6a21\u578b\u6784\u5efa\u591a\u6a21\u6001\u6a21\u578b\uff0c\u4ee5\u5145\u5206\u5229\u7528SOTA\u7684\u89c6\u89c9\u4e0e\u6587\u672cencoder\u7684\u5404\u81ea\u4f18\u52bf\u3002<\/p>\n<pre><code>def __init__(self, config=None, vision_model=None, text_model=None):\n    ...\n    # \u7075\u6d3b\u521d\u59cb\u5316\uff1a\u652f\u6301\u914d\u7f6e\u548c\u6a21\u578b\u7684\u591a\u79cd\u7ec4\u5408\n    if config is None and (vision_model is None or text_model is None):\n        raise ValueError(&quot;Either a configuration or an vision and a text model has to be provided&quot;)\n\n    # \u4ece\u9884\u8bad\u7ec3\u6a21\u578b\u81ea\u52a8\u751f\u6210\u914d\u7f6e\n    if config is None:\n        config = EvlConfig.from_vision_text_configs(vision_model.config, text_model.config)\n\n    # \u6839\u636e\u914d\u7f6e\u7c7b\u578b\u9009\u62e9\u5408\u9002\u7684\u89c6\u89c9\u6a21\u578b\n    if vision_model is None:\n        if isinstance(config.vision_config, CLIPVisionConfig):\n            vision_model = CLIPVisionModel(config.vision_config)\n        elif isinstance(config.vision_config, InternVisionConfig):\n            vision_model = InternVisionModel(config.vision_config)\n    ...\n\n@classmethod\ndef from_vision_text_pretrained(\n    cls, \n    vision_model_name_or_path: str, \n    text_model_name_or_path: str, \n    **kwargs\n):\n    # \u5206\u5e03\u5f0f\u4e0b\u8f7d\u534f\u8c03\n    local_rank = int(os.getenv(&#039;LOCAL_PROCESS_RANK&#039;, &#039;0&#039;))\n    if local_rank == 0:\n        repo_download(repo_id=vision_model_name_or_path, local_dir=vis_load_dir)\n        repo_download(repo_id=text_model_name_or_path, local_dir=text_load_dir)\n        done_file.touch()\n\n    # \u5206\u522b\u52a0\u8f7d\u89c6\u89c9\u548c\u6587\u672c\u6a21\u578b\n    vision_model = CLIPVisionModel.from_pretrained(vis_load_dir, **kwargs_vision)\n    text_model = AutoModel.from_pretrained(text_load_dir, **kwargs_text)\n\n    # \u878d\u5408\u914d\u7f6e\u5e76\u521d\u59cb\u5316\u5b8c\u6574\u6a21\u578b\n    config = EvlConfig.from_vision_text_configs(\n        vision_model.config, \n        text_model.config, \n        **kwargs\n    )\n    model = cls(config=config, vision_model=vision_model, text_model=text_model)\n\n    ...\n\n    # \u53c2\u6570\u4e00\u81f4\u6027\u9a8c\u8bc1\n    print(&quot;vision parameter allclose:&quot;, torch.allclose(\n        vision_model.embeddings.patch_embedding.bias.data,\n        model.vision_model.embeddings.patch_embedding.bias.data\n    ))\n\n    return model, image_processor, tokenizer, processor<\/code><\/pre>\n<p><strong>\u89c6\u89c9\u538b\u7f29\u5c42\u4f18\u5316\u4e0e\u591a\u56fe\u50cf\u5904\u7406<\/strong><\/p>\n<p>\u901a\u8fc7\u50cf\u7d20\u91cd\u6392(pixel shuffle)\u5b9e\u73b0\u7279\u5f81\u538b\u7f29\uff0c\u5c06\u7a7a\u95f4\u7ef4\u5ea6\u4fe1\u606f\u65e0\u635f\u91cd\u7ec4\u5230\u901a\u9053\u7ef4\u5ea6\uff1a\u7a7a\u95f4\u5206\u8fa8\u7387\u4eceH\u00d7W\u964d\u81f3H\/2\u00d7W\/2\uff0c\u76f8\u5e94\u5730\u901a\u9053\u7ef4\u5ea6\u6269\u5c554\u500d\u3002\u5bf9\u4e8e\u591a\u56fe\u50cf\u8f93\u5165\uff0c\u91c7\u7528\u7a7a\u95f4\u7ef4\u5ea6\u6c60\u5316\u548c\u6279\u6b21\u7ef4\u5ea6\u5e73\u5747\u878d\u5408\u7684\u7b56\u7565\u8fdb\u884c\u5904\u7406\u3002<\/p>\n<p><strong>\u4ef7\u503c<\/strong>\uff1a\u51cf\u5c1175%\u7684\u89c6\u89c9token\u6570\u91cf\uff0c\u4e3a\u8ba1\u7b97\u590d\u6742\u5ea6\u4e3aO(n\u00b2d)\u7684\u81ea\u6ce8\u610f\u529b\u673a\u5236\u5e26\u6765\u5e73\u65b9\u7ea7\u7684\u6027\u80fd\u63d0\u5347\uff0c\u540c\u65f6\u9002\u5e94\u591a\u56fe\u573a\u666f\uff0c\u63d0\u5347\u8bad\u7ec3\u548c\u63a8\u7406\u6548\u7387\u3002<\/p>\n<pre><code>def pixel_shuffle(self, x, scale_factor=0.5):\n    &quot;&quot;&quot;\u901a\u8fc7\u591a\u6b65\u7ef4\u5ea6\u53d8\u6362\u5c06\u7a7a\u95f4\u4fe1\u606f\u91cd\u65b0\u5206\u5e03\u5230\u901a\u9053\u7ef4\u5ea6&quot;&quot;&quot;\n    n, w, h, c = x.size()\n\n    # \u7b2c\u4e00\u6b65\u53d8\u6362\uff1a\u8c03\u6574\u9ad8\u5ea6\u548c\u901a\u9053\u7ef4\u5ea6\n    x = x.view(n, w, int(h * scale_factor), int(c \/ scale_factor))\n    x = x.permute(0, 2, 1, 3).contiguous()\n\n    # \u7b2c\u4e8c\u6b65\u53d8\u6362\uff1a\u8c03\u6574\u5bbd\u5ea6\u548c\u901a\u9053\u7ef4\u5ea6\n    x = x.view(n, int(h * scale_factor), int(w * scale_factor), \n               int(c \/ (scale_factor * scale_factor)))\n\n    return x.permute(0, 2, 1, 3).contiguous()\n\ndef extract_feature(self, pixel_values):\n    # \u63d0\u53d6\u89c6\u89c9\u7279\u5f81\n    vit_embeds = self.vision_model(pixel_values=pixel_values).last_hidden_state[:, 1:, :]\n\n    # \u91cd\u5851\u4e3a2D\u7a7a\u95f4\u683c\u5f0f\n    h = w = int(vit_embeds.shape[1] ** 0.5)\n    vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)\n\n    # \u5e94\u7528\u50cf\u7d20\u91cd\u6392\u548c\u4e0b\u91c7\u6837\n    vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)\n\n    # \u5e94\u7528\u89c6\u89c9\u6295\u5f71\u5e76\u6c60\u5316\n    vit_embeds = self.visual_projection(\n        vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])\n    )\n\n    return vit_embeds.mean(dim=1)  # \u7a7a\u95f4\u7ef4\u5ea6\u6c60\u5316<\/code><\/pre>\n<p><strong>\u6587\u672c\u7279\u5f81\u63d0\u53d6\u4e0eT5\u878d\u5408\u67b6\u6784<\/strong><\/p>\n<p>\u901a\u8fc7last_token_pool()\u5b9e\u73b0\u6587\u672c\u5e8f\u5217\u6700\u540e\u6709\u6548token\u7684\u6c60\u5316\uff0c\u81ea\u9002\u5e94\u5904\u7406\u5de6\u53f3padding\u683c\u5f0f\u3002\u6587\u672c\u7279\u5f81\u7ecf\u8fc7\u7ebf\u6027\u6295\u5f71\u5c42\u6620\u5c04\u5230\u5171\u4eab\u8bed\u4e49\u7a7a\u95f4\u3002\u91c7\u7528T5FusionLayer\u5b9e\u73b0\u6df1\u5ea6\u8de8\u6a21\u6001\u878d\u5408\uff0c\u914d\u7f6e2\u5c42Transformer\u300112\u4e2a\u6ce8\u610f\u529b\u5934\uff0c\u5c06\u6587\u672c\u548c\u56fe\u50cf\u7279\u5f81\u62fc\u63a5\u540e\u5229\u7528\u81ea\u6ce8\u610f\u529b\u673a\u5236\u5b66\u4e60\u8de8\u6a21\u6001\u4f9d\u8d56\u5173\u7cfb\u3002<\/p>\n<p><strong>\u4ef7\u503c**<\/strong>\uff1a**\u786e\u4fdd\u6587\u672c\u7279\u5f81\u63d0\u53d6\u51c6\u786e\u6027\uff0c\u652f\u6301\u4e0d\u540c\u6587\u672c\u8f93\u5165\u683c\u5f0f\u3002\u76f8\u6bd4\u7b80\u5355MLP\u878d\u5408\uff0cT5\u67b6\u6784\u80fd\u5b66\u4e60\u66f4\u590d\u6742\u7684\u8de8\u6a21\u6001\u4f9d\u8d56\uff0c\u63d0\u5347\u68c0\u7d22\u548c\u7406\u89e3\u80fd\u529b\u3002<\/p>\n<pre><code>def last_token_pool(self, last_hidden_states: Tensor, attention_mask: Tensor) -&gt; Tensor:\n    &quot;&quot;&quot;\u63d0\u53d6\u5e8f\u5217\u6700\u540e\u4e00\u4e2a\u6709\u6548token\u7684\u7279\u5f81\uff0c\u9002\u914d\u4e0d\u540cpadding\u65b9\u5f0f&quot;&quot;&quot;\n    left_padding = (attention_mask[:, -1].sum() == attention_mask.shape[0])\n\n    if left_padding:\n        return last_hidden_states[:, -1]\n    else:\n        sequence_lengths = attention_mask.sum(dim=1) - 1\n        batch_size = last_hidden_states.shape[0]\n        return last_hidden_states[\n            torch.arange(batch_size, device=last_hidden_states.device), \n            sequence_lengths\n        ]\n\n# \u6587\u672c\u6295\u5f71\u5c42\uff1a\u76f4\u63a5\u7ebf\u6027\u6620\u5c04\u5230\u5171\u4eab\u7a7a\u95f4\nself.text_projection = nn.Linear(\n    self.text_embed_dim, \n    self.projection_dim, \n    bias=False\n)\n\nclass T5FusionLayer(nn.Module):\n    def __init__(self, feature_dim: int):\n        super().__init__()\n        conf_t5 = T5Config()\n        conf_t5.num_layers = 2\n        conf_t5.num_heads = 12\n        conf_t5.d_model = feature_dim\n        conf_t5.d_kv = 64\n\n        self.t5_layers = T5Stack(conf_t5)\n\n    def forward(self, txt_feat, img_feat):\n        # \u62fc\u63a5\u4e3a\u5e8f\u5217\u8f93\u5165\uff0c\u5229\u7528\u81ea\u6ce8\u610f\u529b\u5b66\u4e60\u8de8\u6a21\u6001\u4ea4\u4e92\n        combined_features = torch.cat(\n            [txt_feat.unsqueeze(1), img_feat.unsqueeze(1)], \n            dim=1\n        )\n\n        transformer_output = self.t5_layers(\n            inputs_embeds=combined_features,\n            attention_mask=None, \n            use_cache=False, \n            return_dict=True\n        )\n\n        return torch.mean(transformer_output.last_hidden_state, dim=1)<\/code><\/pre>\n<p><strong>\u6761\u4ef6\u5316\u591a\u4efb\u52a1\u8bad\u7ec3\u7b56\u7565<\/strong><\/p>\n<p>\u5efa\u6a21\u65f6\u5728\u6a21\u578b<code>forward<\/code> \u51fd\u6570\u4e2d\u6839\u636e\u8f93\u5165\u6570\u636e\u7c7b\u578b\u52a8\u6001\u9009\u62e9\u8bad\u7ec3\u6a21\u5f0f\u3002\u5b58\u5728query\u65f6\u6267\u884c\u4e09\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60(query-text\/image\/fusion)\uff0c\u5426\u5219\u6267\u884c\u53cc\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60(text-image)\u3002\u901a\u8fc7loss_weight\u53c2\u6570\u5e73\u8861\u4e0d\u540c\u4efb\u52a1\u8d21\u732e\u5ea6\uff0c\u91c7\u7528\u53cc\u5411\u5bf9\u79f0CLIP\u635f\u5931\u786e\u4fdd\u8bad\u7ec3\u7a33\u5b9a\u6027\u3002<\/p>\n<p><strong>\u4ef7\u503c<\/strong>\uff1a\u5b9e\u73b0\u7edf\u4e00\u6846\u67b6\u4e0b\u7684\u591a\u9636\u6bb5\u8bad\u7ec3\uff0c\u652f\u6301\u9884\u8bad\u7ec3\u5230\u5fae\u8c03\u7684\u65e0\u7f1d\u5207\u6362\u3002<\/p>\n<pre><code>def clip_loss(similarity: torch.Tensor) -&gt; torch.Tensor:\n    &quot;&quot;&quot;\u53cc\u5411\u5bf9\u79f0CLIP\u635f\u5931&quot;&quot;&quot;\n    q2t_loss = contrastive_loss(similarity)\n    t2q_loss = contrastive_loss(similarity.t())\n    return (q2t_loss + t2q_loss) \/ 2.0\n\ndef forward(...):\n    ...\n    # \u6761\u4ef6\u5316\u8bad\u7ec3\u7b56\u7565\n    if query_input_ids is not None:\n        # \u4e09\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60\u6a21\u5f0f\uff08\u5fae\u8c03\u9636\u6bb5\uff09\n        logits_q2t = torch.matmul(query_embeds, text_embeds.t()) * logit_scale\n        logits_q2i = torch.matmul(query_embeds, image_embeds.t()) * logit_scale\n        logits_q2f = torch.matmul(query_embeds, fusion_features.t()) * logit_scale\n\n        loss = (\n            self.loss_weight[0] * clip_loss(logits_q2t) +\n            self.loss_weight[1] * clip_loss(logits_q2i) +\n            self.loss_weight[2] * clip_loss(logits_q2f)\n        )\n    else:\n        # \u53cc\u6a21\u6001\u5bf9\u6bd4\u5b66\u4e60\u6a21\u5f0f\uff08\u9884\u8bad\u7ec3\u9636\u6bb5\uff09\n        logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale\n        loss = clip_loss(logits_per_text)\n    ...<\/code><\/pre>\n<p><strong>\u7279\u5f81\u6807\u51c6\u5316\u4e0e\u6e29\u5ea6\u7f29\u653e<\/strong><\/p>\n<p>\u5bf9\u6240\u6709\u6a21\u6001\u7279\u5f81\u8fdb\u884cL2\u5f52\u4e00\u5316\u786e\u4fdd\u5355\u4f4d\u7403\u9762\u5206\u5e03\uff0c\u901a\u8fc7\u53ef\u5b66\u4e60\u7684logit_scale\u53c2\u6570\u8c03\u8282\u76f8\u4f3c\u5ea6\u5206\u5e03\u9510\u5ea6\u3002\u6307\u6570\u53d8\u6362\u786e\u4fddscale\u4e3a\u6b63\u503c\u3002<\/p>\n<p><strong>\u4ef7\u503c<\/strong>\uff1a\u786e\u4fdd\u4f59\u5f26\u76f8\u4f3c\u5ea6\u8ba1\u7b97\u7a33\u5b9a\u6027\uff0c\u63d0\u5347\u5bf9\u6bd4\u5b66\u4e60\u6548\u679c\u3002<\/p>\n<pre><code># L2\u5f52\u4e00\u5316 + \u53ef\u5b66\u4e60\u6e29\u5ea6\u7f29\u653e\ntext_embeds = text_embeds \/ text_embeds.norm(dim=-1, keepdim=True)\nimage_embeds = image_embeds \/ image_embeds.norm(dim=-1, keepdim=True)\nfusion_features = fusion_features \/ fusion_features.norm(dim=-1, keepdim=True)\n\nlogit_scale = self.logit_scale.exp()  # \u786e\u4fdd\u4e3a\u6b63\u503c\nlogits = torch.matmul(query_embeds, text_embeds.t()) * logit_scale<\/code><\/pre>\n<p><strong>\u7ed3\u6784\u5316\u8f93\u51fa\u7c7b\u8bbe\u8ba1<\/strong><\/p>\n<p>SFTOutput\u7ee7\u627fCLIPOutput\u5e76\u6269\u5c55\u591a\u6a21\u6001\u8f93\u51fa\u5b57\u6bb5\uff0c\u4fdd\u5b58\u6240\u6709\u4e2d\u95f4\u7279\u5f81\u548c\u76f8\u4f3c\u5ea6\u77e9\u9635\uff0c\u652f\u6301\u7075\u6d3b\u7684\u8fd4\u56de\u683c\u5f0f\u3002<\/p>\n<p><strong>\u4ef7\u503c**<\/strong>\uff1a**\u63d0\u4f9b\u5b8c\u6574\u7684\u6a21\u578b\u8f93\u51fa\u4fe1\u606f\uff0c\u4fbf\u4e8e\u6a21\u578b\u5206\u6790\u3001\u8c03\u8bd5\u548c\u4e0b\u6e38\u4efb\u52a1\u4f7f\u7528\u3002<\/p>\n<pre><code>@dataclass\nclass SFTStage2Output(CLIPOutput):\n    &quot;&quot;&quot;\u6269\u5c55\u7684\u591a\u6a21\u6001\u8f93\u51fa\u7c7b\uff0c\u4fdd\u5b58\u5b8c\u6574\u7684\u4e2d\u95f4\u7ed3\u679c&quot;&quot;&quot;\n\n    loss: Optional[torch.FloatTensor] = None\n    logits_q2t: torch.FloatTensor = None  # query-text\u76f8\u4f3c\u5ea6\n    logits_q2i: torch.FloatTensor = None  # query-image\u76f8\u4f3c\u5ea6\n    logits_q2f: torch.FloatTensor = None  # query-fusion\u76f8\u4f3c\u5ea6\n    text_embeds: torch.FloatTensor = None\n    image_embeds: torch.FloatTensor = None\n    fusion_features: torch.FloatTensor = None\n\n    def to_tuple(self) -&gt; Tuple[Any]:\n        &quot;&quot;&quot;\u5c06\u6570\u636e\u7c7b\u5b9e\u4f8b\u8f6c\u6362\u4e3a\u5143\u7ec4\uff0c\u5904\u7406\u5d4c\u5957\u5bf9\u8c61&quot;&quot;&quot;\n        return tuple(\n            self[k] if k not in [&quot;text_output&quot;, &quot;vision_output&quot;, &quot;query_output&quot;]\n            else getattr(self, k).to_tuple()\n            for k in self.keys()\n        )<\/code><\/pre>\n<p><strong>\u8bad\u7ec3\u8d85\u53c2\u8bbe\u7f6e<\/strong><\/p>\n<pre><code>\u25cf SFT\uff1a\u5355\u5361bs=8\uff0cga=36\uff0clr=2e-5\uff08\u4f59\u5f26\u9000\u706b\uff0cwarmup0.05\uff09\uff0cweight_decay=0.1\uff0cmax_grad_norm 1.0\uff0cadam\u53c2\u6570\uff080.9,0.95,1e-5\uff0c\u548c\u9ed8\u8ba4\u503c\u533a\u522b\u4e0d\u5927\uff09    \n  \u25cb 128\u5361\uff0c1400w\u6837\u672c\uff0c1600step\uff0c\u7ea64\u4e2aepoch\uff0c\u8bad\u7ec3\u65f6\u95f44.5\u5929        \n    \u25a0 per_N = N\/world_size        \n    \u25a0 max_steps\uff081epoch\uff09 = per_N\/(\u5355\u5361bs * ga)        \n    \u25a0 \u8fd9\u91ccbs ga\u5c1d\u8bd5\u4e86\u4e0d\u540c\u7ec4\u5408\u65e8\u5728\u903c\u8fd1gpu\u5229\u7528\u7387\u4e0a\u9650    \n  \u25cb weight_decay\uff1a        \n    \u25a0 \u5373l2\u6b63\u5219\u5316\uff0c\u76ee\u7684\u662f\u8ba9\u6743\u91cd\u8870\u51cf\u5230\u66f4\u5c0f\u7684\u503c\uff0c\u5728\u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u51cf\u5c11\u6a21\u578b\u8fc7\u62df\u5408\u7684\u95ee\u9898        \n    \u25a0 \u66f4\u5c0f\u7684\u6743\u503cw\uff0c\u4ece\u67d0\u79cd\u610f\u4e49\u4e0a\u8bf4\uff0c\u8868\u793a\u7f51\u7edc\u7684\u590d\u6742\u5ea6\u66f4\u4f4e\uff0c\u5bf9\u6570\u636e\u7684\u62df\u5408\u66f4\u597d\uff08\u8fd9\u4e2a\u6cd5\u5219\u4e5f\u53eb\u505a\u5965\u5361\u59c6\u5243\u5200\uff09\uff0c\u8fd9\u91cc\u62cd\u4e86\u4e2a\u5e38\u7528\u503c    \n  \u25cb \u68af\u5ea6\u88c1\u526a\uff08\u9632\u6b62\u68af\u5ea6\u7206\u70b8\uff09\uff1a        \n    \u25a0 new_grad = grad * max_grad_norm \/ max(grad_norm, max_grad_norm)        \n    \u25a0 grad_norm\u4e3a\u6240\u6709\u53c2\u6570\u7684\u68af\u5ea6\u7684L2\u8303\u6570\n\u25cf PT\u53c2\u6570\u57fa\u672c\u4e00\u81f4\uff0c\u5c31bs ga\u7565\u5927\u4e86\u4e9b\uff08bs=24,ga=16\uff0c\u6bd5\u7adf\u5c11\u4e86\u878d\u5408\u6a21\u5757\u548c\u591a\u4e2aloss\uff09\uff0c\u4e5f\u662f4\u4e2aepoch\uff0c873w\u6837\u672c\uff0c\u7ea62\u5929<\/code><\/pre>\n<h3>3.5.2 \u5de5\u7a0b\u5b9e\u73b0<\/h3>\n<p><strong>\u52a8\u6001\u5206\u8fa8\u7387\u9884\u5904\u7406\u7ba1\u9053<\/strong><\/p>\n<p>\u57fa\u4e8e\u5bbd\u9ad8\u6bd4\u7684\u81ea\u9002\u5e94\u56fe\u50cf\u5207\u5206\u7b97\u6cd5\uff0c\u8ba1\u7b97\u6700\u4f18\u5207\u5206\u65b9\u6848\u4fdd\u6301\u56fe\u50cf\u7ec6\u8282\uff0c\u6dfb\u52a0\u7f29\u7565\u56fe\u7ef4\u6301\u5168\u5c40\u4fe1\u606f\u3002<\/p>\n<p><strong>\u4ef7\u503c**<\/strong>\uff1a**\u5e73\u8861\u56fe\u50cf\u8bed\u4e49\u5b8c\u6574\u6027\u4e0e\u8ba1\u7b97\u6548\u7387\uff0c\u9002\u5e94\u4e0d\u540c\u5c3a\u5bf8\u56fe\u50cf\u7684\u5904\u7406\u9700\u6c42\u3002<\/p>\n<pre><code>def dynamic_preprocess(image, min_num=1, max_num=12, image_size=448, use_thumbnail=False):\n    &quot;&quot;&quot;\u57fa\u4e8e\u5bbd\u9ad8\u6bd4\u7684\u81ea\u9002\u5e94\u56fe\u50cf\u5207\u5206&quot;&quot;&quot;\n    # \u8ba1\u7b97\u6700\u4f18\u5207\u5206\u6bd4\u4f8b\n    target_ratios = set(\n        (i, j) for n in range(min_num, max_num + 1)\n        for i in range(1, n + 1) \n        for j in range(1, n + 1)\n        if i * j &lt;= max_num and i * j &gt;= min_num\n    )\n\n    target_aspect_ratio = find_closest_aspect_ratio(\n        aspect_ratio, target_ratios, ...\n    )\n    ...\n\n    # \u6309\u6700\u4f18\u6bd4\u4f8b\u5207\u5206\n    processed_images = []\n    for i in range(target_aspect_ratio[0] * target_aspect_ratio[1]):\n        # \u8ba1\u7b97\u5207\u5206\u533a\u57df\u5e76\u88c1\u526a\n        ...\n        processed_images.append(resized_img.crop(box))\n\n    # \u6dfb\u52a0\u7f29\u7565\u56fe\u4fdd\u6301\u5168\u5c40\u4e0a\u4e0b\u6587\n    if use_thumbnail and len(processed_images) != 1:\n        processed_images.append(image.resize((image_size, image_size)))\n\n    ...\n    return processed_images<\/code><\/pre>\n<p><strong>\u6d41\u5f0f\u6570\u636e\u52a0\u8f7d\u9002\u914d<\/strong><\/p>\n<p>transformers Trainer\u9ed8\u8ba4\u671f\u671b\u6570\u636e\u96c6\u5b9e\u73b0<code>__len__<\/code>\u65b9\u6cd5\uff0c\u6d41\u5f0f\u6570\u636e\u52a0\u8f7d\u9700\u8981\u81ea\u5b9a\u4e49Trainer\u9002\u914dIterableDataset\uff0c\u652f\u6301\u5206\u5e03\u5f0f\u73af\u5883\u4e0b\u7684\u6570\u636e\u6d41\u5904\u7406\u3002<\/p>\n<p><strong>\u4ef7\u503c**<\/strong>\uff1a**\u5b9e\u73b0\u5927\u89c4\u6a21\u6570\u636e\u6d41\u5f0f\u52a0\u8f7d\uff0c\u907f\u514d\u5185\u5b58\u6ea2\u51fa\u3002<\/p>\n<pre><code>class MyTrainer(Trainer):\n    def get_train_dataloader(self) -&gt; DataLoader:\n        if isinstance(train_dataset, torch.utils.data.IterableDataset):\n            return DataLoader(\n                train_dataset,\n                batch_size=self._train_batch_size,\n                collate_fn=data_collator,\n                drop_last=True,\n                num_workers=self.args.dataloader_num_workers,\n                pin_memory=self.args.dataloader_pin_memory\n            )<\/code><\/pre>\n<p><strong>\u9ad8\u6548\u6570\u636e\u6536\u96c6\u5668\u8bbe\u8ba1<\/strong><\/p>\n<p>transformers Trainer\u4e2d\uff0cdata_collator\u7528\u4e8e\u5c06dataset\u4e2d\u4e00\u4e2abatch\u7684\u8f93\u5165\u6570\u636e\u8f6c\u6362\u4e3a\u6a21\u578b\u4e00\u4e2abatch\u8f93\u5165\u7684\u5f62\u5f0f\uff0c\u5b9e\u73b0\u81ea\u5b9a\u4e49DataCollator\u5904\u7406\u591a\u6a21\u6001\u8f93\u5165\u7684\u52a8\u6001padding\uff0c\u53ef\u6839\u636e\u8bad\u7ec3\u7c7b\u578b(\u9884\u8bad\u7ec3\/\u5fae\u8c03)\u7075\u6d3b\u5904\u7406query\u8f93\u5165\uff0c\u786e\u4fdd\u6279\u6b21\u5185\u6570\u636e\u683c\u5f0f\u4e00\u81f4\u3002<\/p>\n<p><strong>\u4ef7\u503c<\/strong>\uff1a\u63d0\u5347\u6570\u636e\u52a0\u8f7d\u6548\u7387\uff0c\u652f\u6301\u4e0d\u540c\u8bad\u7ec3\u9636\u6bb5\u7684\u6570\u636e\u683c\u5f0f\u9700\u6c42\u3002<\/p>\n<pre><code>@dataclass\nclass MyDataCollator(object):\n    &quot;&quot;&quot;\u591a\u6a21\u6001\u6570\u636e\u6536\u96c6\u5668\uff0c\u652f\u6301\u52a8\u6001padding\u548c\u8bad\u7ec3\u7c7b\u578b\u9002\u914d&quot;&quot;&quot;\n\n    def __call__(self, instances: Sequence[Dict]) -&gt; Dict[str, torch.Tensor]:\n        # \u6839\u636e\u8bad\u7ec3\u7c7b\u578b\u5904\u7406query\n        if self.train_type == &quot;sft&quot;:\n            query_input_ids = torch.nn.utils.rnn.pad_sequence(\n                query_input_ids, \n                batch_first=True, \n                padding_value=self.tokenizer.pad_token_id\n            )\n            query_attention_mask = query_input_ids.ne(self.tokenizer.pad_token_id)\n        else:\n            query_input_ids, query_attention_mask = None, None\n\n        # \u6587\u672c\u52a8\u6001padding\n        text_input_ids = torch.nn.utils.rnn.pad_sequence(\n            text_input_ids, \n            batch_first=True, \n            padding_value=self.tokenizer.pad_token_id\n        )\n        text_attention_mask = text_input_ids.ne(self.tokenizer.pad_token_id)\n\n        return dict(\n            query_input_ids=query_input_ids,\n            query_attention_mask=query_attention_mask,\n            text_input_ids=text_input_ids,\n            text_attention_mask=text_attention_mask,\n            pixel_values=pixel_values,\n            return_loss=self.return_loss\n        )<\/code><\/pre>\n<p><strong>\u591a\u7ebf\u7a0b\u56fe\u50cf\u62c9\u53d6\u4f18\u5316<\/strong><\/p>\n<p>\u914d\u7f6e8\u4e2a\u56fe\u50cf\u62c9\u53d6\u7ebf\u7a0b\u548c16\u4e2a\u6570\u636e\u8bfb\u53d6\u7ebf\u7a0b\uff0c\u652f\u6301\u7f51\u7edc\u5f02\u5e38\u91cd\u8bd5\u673a\u5236\uff08\u6700\u591a10\u6b21\uff0c\u95f4\u96945\u79d2\uff09\u3002<\/p>\n<p><strong>\u4ef7\u503c<\/strong>\uff1a\u663e\u8457\u63d0\u5347\u56fe\u50cf\u52a0\u8f7d\u901f\u5ea6\uff0c\u51cf\u5c11IO\u7b49\u5f85\u65f6\u95f4\u3002<\/p>\n<pre><code># \u591a\u7ebf\u7a0b\u914d\u7f6e\nconvert_args = {    \n  PARALLEL_WORKER_NUM: 8,  # 8\u4e2a\u56fe\u50cf\u62c9\u53d6\u7ebf\u7a0b    \n  &#039;oss_pull_func&#039;: partial(ele_pull_resource_func, image_processor=image_processor)\n}\ntrain_dataset = DistributeDataset(data_args.tables, num_threads=16, capacity=1024, ...)<\/code><\/pre>\n<p><strong>\u5f02\u5e38\u5904\u7406\u4e0e\u5bb9\u9519\u673a\u5236<\/strong><\/p>\n<p>\u5b9e\u73b0\u591a\u5c42\u6b21\u5f02\u5e38\u5904\u7406\uff0c\u5305\u62ec\u7f51\u7edc\u91cd\u8bd5\u3001\u6570\u636e\u5b8c\u6574\u6027\u68c0\u67e5\u3001\u56fe\u50cf\u683c\u5f0f\u8f6c\u6362\u5f02\u5e38\u6355\u83b7\u3002\u5931\u8d25\u65f6\u751f\u6210\u968f\u673atensor\u515c\u5e95\uff0c\u786e\u4fdd\u8bad\u7ec3\u4e0d\u4e2d\u65ad\u3002<\/p>\n<p><strong>\u4ef7\u503c<\/strong>\uff1a\u63d0\u5347\u8bad\u7ec3\u7a33\u5b9a\u6027\uff0c\u4fdd\u8bc1\u5927\u89c4\u6a21\u8bad\u7ec3\u53ef\u9760\u6027\u3002<\/p>\n<pre><code># \u7f51\u7edc\u91cd\u8bd5 + \u515c\u5e95\u673a\u5236\ndef url2image_pixel(image_urls_str, image_processor, max_sub_img_num=10):\n    pix_list = []\n\n    for sub_url in sub_images[:max_sub_img_num]:\n        timeout_counter = 0\n        response = &quot;&quot;\n\n        while not response and timeout_counter &lt;= 10:\n            try:\n                response = requests.get(sub_url, stream=True).raw\n                if response:\n                    pixel_value_i = image_processor(\n                        images=Image.open(response).convert(&quot;RGB&quot;),\n                        return_tensors=&quot;pt&quot;\n                    ).pixel_values\n                    pix_list.append(pixel_value_i)\n            except Exception:\n                timeout_counter += 1\n                sleep(5)\n\n    # \u515c\u5e95\u5904\u7406\n    if not pix_list:\n        print(&quot;cannot get right image, fake one!!&quot;)\n        pix_list.append(torch.rand(1, 3, 448, 448))\n\n    return torch.cat(pix_list, dim=0)<\/code><\/pre>\n<p><strong>\u5206\u5e03\u5f0f\u8bad\u7ec3\u534f\u8c03<\/strong><\/p>\n<p>mdl\u8bad\u7ec3\u65f6\uff0c\u4f7f\u7528rank 0\u7edf\u4e00\u4e0b\u8f7d\u6a21\u578b\uff0c\u5176\u4ed6\u8fdb\u7a0b\u7b49\u5f85\u5b8c\u6210\u4fe1\u53f7\uff0c\u901a\u8fc7torch.distributed.barrier()\u5b9e\u73b0\u540c\u6b65\u3002<\/p>\n<p><strong>\u4ef7\u503c**<\/strong>\uff1a**\u907f\u514d\u91cd\u590d\u4e0b\u8f7d\uff0c\u786e\u4fdd\u5206\u5e03\u5f0f\u73af\u5883\u4e00\u81f4\u6027\u3002<\/p>\n<pre><code>local_rank = int(os.getenv(&quot;LOCAL_PROCESS_RANK&quot;, &quot;0&quot;))\n\nif local_rank == 0:\n    repo_download(repo_id=mos_model_uri, local_dir=load_dir)\n    done_file.touch()\n\nif torch.distributed.is_initialized():\n    torch.distributed.barrier()\nelse:\n    while not os.path.exists(done_file):\n        time.sleep(60)<\/code><\/pre>\n<h3>3.5.3 \u8e29\u5751\u7ecf\u9a8c<\/h3>\n<p><strong>Config\u7c7b\u517c\u5bb9\u6027\u95ee\u9898<\/strong><\/p>\n<p><strong>\u95ee\u9898\uff1a<\/strong>\u81ea\u5b9a\u4e49Config\u7c7b\u5728\u8c03\u7528<code>save_pretrained()<\/code>\u65f6\u62a5\u9519\uff0c\u63d0\u793a\u5fc5\u9700\u53c2\u6570\u4e3a<code>None<\/code>\u3002<\/p>\n<p><strong>\u6839\u672c\u539f\u56e0<\/strong>\uff1atransformers\u5e93\u5728\u8c03\u7528<code>save_pretrained<\/code>\u65f6\uff0c\u4e3a\u4e86\u7cbe\u7b80config\u53c2\u6570\uff0c\u4f1a\u7528\u9ed8\u8ba4\u53c2\u6570\u521b\u5efa\u65b0config\u5b9e\u4f8b\u8fdb\u884c\u5bf9\u6bd4\uff0c\u53ea\u5b58\u50a8\u4e0d\u540c\u9879\u3002\u8fd9\u4f1a\u5bfc\u81f4\u81ea\u52a8\u5316\u5730\u4f7f\u7528\u9ed8\u8ba4\u53c2\u6570\u8c03\u7528config\u7c7b\u7684<code>__init__<\/code>\u51fd\u6570\u3002<\/p>\n<p><strong>\u89e3\u51b3\u65b9\u6848<\/strong>\uff1a<\/p>\n<pre><code>class DualConfig(PretrainedConfig):\n    model_type = &quot;dual_model&quot;\n    is_composition = True  # \u4f18\u5148\u65b9\u6848\uff1a\u6807\u8bc6\u4e3a\u590d\u5408\u914d\u7f6e\n\n    def __init__(self, projection_dim=512, **kwargs):\n        super().__init__(**kwargs)\n\n        # \u515c\u5e95\u65b9\u6848\uff1a\u652f\u6301\u9ed8\u8ba4\u521d\u59cb\u5316\n        vision_config = kwargs.pop(&quot;vision_config&quot;, None)\n        text_config = kwargs.pop(&quot;text_config&quot;, None)\n\n        self.vision_config = vision_config if vision_config else ViTConfig()\n        self.text_config = text_config if text_config else BertConfig()\n        self.projection_dim = projection_dim<\/code><\/pre>\n<p>``<\/p>\n<p><strong>\u7ecf\u9a8c\u8981\u70b9<\/strong>\uff1a<\/p>\n<ul>\n<li><strong>\u4f18\u5148\u65b9\u6848**<\/strong>\uff1a**\u8bbe\u7f6e<code>is_composition = True<\/code>\uff0c\u544a\u77e5transformers\u8fd9\u662f\u590d\u5408\u914d\u7f6e\uff0c\u8df3\u8fc7\u9ed8\u8ba4\u521d\u59cb\u5316\u5bf9\u6bd4\uff1b<\/li>\n<li><strong>\u515c\u5e95\u65b9\u6848**<\/strong>\uff1a**\u5f53transformers\u7248\u672c\u4e0d\u652f\u6301<code>is_composition<\/code>\u65f6\uff0c\u4fee\u6539<code>__init__<\/code>\u65b9\u6cd5\u652f\u6301\u65e0\u53c2\u6570\u521d\u59cb\u5316\uff1b<\/li>\n<li><strong>\u9002\u7528\u573a\u666f**<\/strong>\uff1a**\u6240\u6709\u5305\u542b\u5b50\u914d\u7f6e\u7684\u81ea\u5b9a\u4e49PretrainedConfig\u7c7b\uff08\u7f16\u7801\u5668\u89e3\u7801\u5668\u67b6\u6784\u3001\u53cc\u5854\u67b6\u6784\u7b49\uff09\uff1b<\/li>\n<\/ul>\n<p><strong>DataCollator\u4e0e\u6a21\u578bForward\u53c2\u6570\u5bf9\u9f50\u95ee\u9898<\/strong><\/p>\n<p><strong>\u95ee\u9898\uff1a<\/strong>transformers\u7684Trainer\u4e2d\uff0c<code>data_collator<\/code>\u8fd4\u56de\u7684\u5b57\u5178key\u5fc5\u987b\u4e0e\u6a21\u578b<code>forward<\/code>\u65b9\u6cd5\u7684\u53c2\u6570\u540d\u4e25\u683c\u5bf9\u5e94\uff0c\u5426\u5219\u4f1a\u51fa\u73b0\u53c2\u6570\u4f20\u9012\u9519\u8bef\u3002<\/p>\n<p><strong>\u6838\u5fc3\u539f\u7406**<\/strong>\uff1a**Trainer\u4f1a\u5c06<code>data_collator<\/code>\u8fd4\u56de\u7684\u5b57\u5178\u76f4\u63a5\u89e3\u5305\u4f20\u9012\u7ed9\u6a21\u578b\u7684<code>forward<\/code>\u65b9\u6cd5\uff1a<\/p>\n<pre><code># Trainer\u5185\u90e8\u903b\u8f91\nbatch = data_collator(samples)  # \u8fd4\u56de {&quot;pixel_values&quot;: ..., &quot;query_input_ids&quot;: ...}\noutputs = model(**batch)        # \u76f4\u63a5\u89e3\u5305\u4f20\u9012<\/code><\/pre>\n<p><strong>\u89e3\u51b3\u65b9\u6848**<\/strong>\uff1a\u786e\u4fdd\u53c2\u6570\u540d\u4e25\u683c\u5bf9\u5e94**<\/p>\n<pre><code># \u6a21\u578bforward\u65b9\u6cd5\u5b9a\u4e49\ndef forward(\n    self,\n    query_input_ids: Optional[torch.LongTensor] = None,\n    text_input_ids: Optional[torch.LongTensor] = None,\n    pixel_values: Optional[Tuple[torch.FloatTensor]] = None,\n    query_attention_mask: Optional[torch.Tensor] = None,\n    text_attention_mask: Optional[torch.Tensor] = None,\n    return_loss: Optional[bool] = None,\n    **kwargs\n):\n    # \u524d\u5411\u4f20\u64ad\u903b\u8f91\n    ...\n\n# DataCollator\u8fd4\u56de\u7684\u5b57\u5178key\u5fc5\u987b\u5bf9\u5e94forward\u53c2\u6570\nclass MyDataCollator(object):\n    def __call__(self, batch) -&gt; Dict:\n        return {\n            &quot;pixel_values&quot;: pixel_values,           # \u2713 \u5bf9\u5e94forward\u53c2\u6570\n            &quot;query_input_ids&quot;: input_ids,           # \u2713 \u5bf9\u5e94forward\u53c2\u6570\n            &quot;query_attention_mask&quot;: attention_mask, # \u2713 \u5bf9\u5e94forward\u53c2\u6570\n            &quot;text_input_ids&quot;: input_ids,            # \u2713 \u5bf9\u5e94forward\u53c2\u6570\n            &quot;text_attention_mask&quot;: attention_mask,  # \u2713 \u5bf9\u5e94forward\u53c2\u6570\n            &quot;return_loss&quot;: True,                    # \u2713 \u5bf9\u5e94forward\u53c2\u6570\n        }<\/code><\/pre>\n<p><strong>\u7ecf\u9a8c\u8981\u70b9<\/strong>\uff1a<\/p>\n<ul>\n<li><strong>\u4e25\u683c\u5bf9\u5e94<\/strong>\uff1aDataCollator\u8fd4\u56de\u5b57\u5178\u7684\u6bcf\u4e2akey\u90fd\u5fc5\u987b\u5728\u6a21\u578bforward\u65b9\u6cd5\u4e2d\u6709\u5bf9\u5e94\u53c2\u6570\uff1b<\/li>\n<li><strong>\u6269\u5c55\u6027<\/strong>\uff1a\u4f7f\u7528<code>**kwargs<\/code>\u63a5\u6536\u989d\u5916\u53c2\u6570\uff0c\u63d0\u9ad8\u4ee3\u7801\u7684\u7075\u6d3b\u6027\uff1b<\/li>\n<\/ul>\n<p><strong>3.6 \u6700\u7ec8\u4e1a\u52a1\u6548\u679c<\/strong><\/p>\n<p>\u6574\u4e2a\u7cfb\u7edf\u91c7\u7528\u5206\u5c42\u90e8\u7f72\u67b6\u6784\uff1aMDL\u5e73\u53f0\u8d1f\u8d23\u53cc\u5854\u6a21\u578b\u8bad\u7ec3\u5e76\u751f\u6210\u591a\u6a21\u6001embedding\uff0cAOP\u5e73\u53f0\u63a5\u5165\u591a\u6a21\u6001\u7279\u5f81\u8fdb\u884cCTR\/CTCVR\u8054\u5408\u8bad\u7ec3\uff0cRTP\u5e73\u53f0\u90e8\u7f72\u6700\u7ec8\u6a21\u578b\u7528\u4e8e\u5728\u7ebf\u63a8\u7406\u670d\u52a1\u3002<\/p>\n<h3>3.6.1\u79bb\u7ebf\u6307\u6807\u63d0\u5347<\/h3>\n<p><strong>\u591a\u6a21\u6001\u9884\u8bad\u7ec3\u6548\u679c**<\/strong>\uff1a**\u7ecf\u8fc7\u9886\u57df\u9002\u914d\u9884\u8bad\u7ec3\u7684\u53cc\u5854\u6a21\u578b\u5728\u8de8\u6a21\u6001\u68c0\u7d22\u4efb\u52a1\u4e0a\u6574\u4f53\u53ec\u56de\u7387\u8fbe\u523090.8%\uff0c\u663e\u8457\u8d85\u8d8aSOTA\u901a\u7528\u6a21\u578bGME-qwen2-vl-7b\uff0866.5%\uff09\u3002<\/p>\n<p><strong>\u7528\u6237\u67e5\u8be2\u5bf9\u9f50\u6548\u679c**<\/strong>\uff1a**\u5fae\u8c03\u540e\u7684EVL-sep\u6a21\u578b\u5728\u641c\u7d22\u8ba2\u5355\u573a\u666f\u8bc4\u4f30\u4e2d\u5e73\u5747\u547d\u4e2d\u7387\u8fbe\u523086.0%\uff0c\u4ec5\u4f7f\u7528128\u7ef4\u8868\u5f81\u5374\u4f18\u4e8e\u9ad8\u7ef4GME\u7cfb\u5217\u6a21\u578b\uff0c\u5728\u5e97\u54c1\u6df7\u5408\u67e5\u8be2\u573a\u666fHR@1\u8fbe\u523053.2%\u3002<\/p>\n<p><strong>\u8054\u5408\u8bad\u7ec3\u6548\u679c**<\/strong>\uff1a**\u7ecf\u8fc7\u591a\u6a21\u6001\u4e0eID\u7279\u5f81\u878d\u5408\u4f18\u5316\uff0c\u63a8\u8350\u7cfb\u7edf\u79bb\u7ebfAUC\u83b7\u5f97\u663e\u8457\u63d0\u5347\uff1a\u5168\u91cf\u5546\u6237CTCVR AUC +0.19%\uff0cCTR AUC +0.17%\u3002<\/p>\n<h3>3.6.2 \u5728\u7ebf\u4e1a\u52a1\u6548\u679c<\/h3>\n<p>\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u4e0a\u7ebf\u540e\u5728\u997f\u4e86\u4e48\u9996\u9875\u5e97\u94fa\u63a8\u8350\u573a\u666f\u53d6\u5f97\u663e\u8457\u4e1a\u52a1\u63d0\u5347\uff1a<\/p>\n<ul>\n<li><strong>\u8ba2\u5355\u91cf\u63d0\u5347 +0.51%<\/strong><\/li>\n<li><strong>\u8bbf\u8d2d\u7387\u63d0\u5347 +0.17%<\/strong><\/li>\n<li><strong>\u51c0GMV\u63d0\u5347 +0.75%<\/strong><\/li>\n<\/ul>\n<p>\u56db\u3001\u5c55\u671b-\u751f\u6210\u5f0f\u63a8\u8350<\/p>\n<p>\u968f\u7740\u5927\u8bed\u8a00\u6a21\u578b\u6280\u672f\u7684\u5feb\u901f\u6210\u719f\uff0c\u751f\u6210\u5f0f\u63a8\u8350\u4f5c\u4e3a\u65b0\u5174\u7684\u6280\u672f\u8303\u5f0f\u5f15\u8d77\u4e86\u4e1a\u754c\u7684\u5e7f\u6cdb\u5173\u6ce8\u3002\u5927\u6a21\u578b\u5728\u8bed\u4e49\u7406\u89e3\u3001\u591a\u6a21\u6001\u4fe1\u606f\u5904\u7406\u548c\u903b\u8f91\u63a8\u7406\u65b9\u9762\u7684\u4f18\u52bf\uff0c\u4e0e\u4f20\u7edf\u63a8\u8350\u7cfb\u7edf\u5728\u534f\u540c\u4fe1\u53f7\u548c\u9886\u57df\u77e5\u8bc6\u65b9\u9762\u7684\u4e13\u957f\u5f62\u6210\u4e86\u5b8c\u7f8e\u4e92\u8865\uff0c\u4e3a\u63a8\u8350\u6280\u672f\u5411\u751f\u6210\u8303\u5f0f\u7684\u6f14\u8fdb\u63d0\u4f9b\u4e86\u53ef\u80fd\u3002<\/p>\n<p><strong>4.1 \u751f\u6210\u5f0f\u5efa\u6a21\u7684\u5c1d\u8bd5<\/strong><\/p>\n<p>\u57fa\u4e8e24\u5e74S2\u5728\u591a\u6a21\u6001\u63a8\u8350\u7684\u6280\u672f\u79ef\u7d2f\uff0c\u9879\u76ee\u7ec4\u81ea25\u5e74\u521d\u5f00\u59cb\u63a2\u7d22\u751f\u6210\u5f0franking\u7684\u5efa\u6a21\u601d\u8def\uff0c\u5c1d\u8bd5\u5c06\u591a\u6a21\u6001\u8868\u5f81\u80fd\u529b\u4e0e\u5927\u8bed\u8a00\u6a21\u578b\u7684\u751f\u6210\u80fd\u529b\u76f8\u7ed3\u5408\uff0c\u7b14\u8005\u4e5f\u6709\u5e78\u53c2\u4e0e\u4e86\u5176\u4e2d\u7684\u521d\u7248\u65b9\u6848\u5efa\u8bbe\u4e4b\u4e2d\u3002\u867d\u7136\u521d\u7248\u65b9\u6848\u7531\u4e8einfra\u65b9\u9762\u95ee\u9898\u843d\u5730\u5b58\u5728\u5361\u70b9\uff0c\u4f46\u4e3a\u7b14\u8005\u7406\u89e3\u751f\u6210\u5f0f\u63a8\u8350\u7684\u6280\u672f\u6311\u6218\u63d0\u4f9b\u4e86\u5b9d\u8d35\u7ecf\u9a8c\uff0c\u5728\u8fd9\u91cc\u4ec5\u5bf9\u5efa\u6a21\u4e0e\u8bad\u7ec3\u601d\u8def\u8fdb\u884c\u5206\u4eab\u3002<\/p>\n<h3>4.1.1 \u7279\u5f81\u4f53\u7cfb\u91cd\u6784\u4e0e\u6570\u636e\u6784\u5efa<\/h3>\n<p><strong>\u63a8\u8350\u7279\u5f81\u4f53\u7cfb\u91cd\u6784\uff1a<\/strong><\/p>\n<p>\u4e3a\u9002\u5e94\u5927\u8bed\u8a00\u6a21\u578b\u7684\u8f93\u5165\u9700\u6c42\uff0c\u6211\u4eec\u5bf9\u4f20\u7edf\u63a8\u8350\u7cfb\u7edf\u7684\u7279\u5f81\u4f53\u7cfb\u8fdb\u884c\u4e86\u5168\u9762\u91cd\u6784\uff0c\u68b3\u7406\u8f6c\u4e49\u63a8\u8350\u5927\u5bbd\u8868\u4e2d\u7684900+\u63a8\u8350\u7ebf\u4e0a\u4e1a\u52a1\u7279\u5f81\uff0c\u6df1\u5165\u5206\u6790\u5404\u4e2a\u7279\u5f81\u5728\u751f\u6210\u5f0f\u8303\u5f0f\u4e0b\u7684\u53ef\u7528\u6027\u4e0e\u4f7f\u7528\u65b9\u6cd5\uff0c\u6309\u7167\u7528\u6237\u7279\u5f81\u3001\u5546\u6237\u7279\u5f81\u3001\u7528\u6237\u5546\u6237\u4ea4\u53c9\u7279\u5f81\u3001\u65f6\u7a7a\u7279\u5f81\u8fdb\u884c\u7b5b\u9009\u5206\u7c7b\uff0c\u6784\u5efa\u9762\u5411llm\u7684\u63a8\u8350\u7279\u5f81\u4f53\u7cfb\u3002<\/p>\n<p><strong>\u5546\u6237\u8868\u5f81\u5efa\u6a21\u7b56\u7565\uff1a<\/strong><\/p>\n<ul>\n<li>\u7528\u6237\u4e2a\u6027\u5316\u5546\u6237\u8868\u5f81\uff08\u9488\u5bf9\u5386\u53f2\u4e0b\u5355\u5e8f\u5217\uff09\uff1a\n<ul>\n<li>\u4ee5user_id+shop_id\u4e3a\u4e3b\u952e\u7ec4\u7ec7\u4e2a\u6027\u5316\u7684\u7528\u6237-\u5546\u6237\u4ea4\u53c9\u878d\u5408\u8868\u5f81\u3002\u5c06\u4f20\u7edf\u7684\u7edf\u8ba1\u7279\u5f81\u8f6c\u5316\u4e3a\u81ea\u7136\u8bed\u8a00\u63cf\u8ff0\uff0c\u5982&quot;x\u5e74x\u6708x\u65e5\u5728xx\u5e97\u94fa\u70b9\u4e86xx\u5546\u54c1\uff0c\u8ba2\u5355\u91d1\u989dxx\u5143\uff0c\u8fd9\u4e2a\u5e97\u94fa\u662fxxx\uff0c\u8fd9\u4e2a\u5546\u54c1\u662fxxx&quot;\u7684cross_shop_token\u683c\u5f0f\uff1b<\/li>\n<li>\u5f15\u5165\u8282\u6c14\u3001\u8282\u65e5\u3001\u661f\u671f\u7b49\u5185\u5bb9\u5c42\u9762\u7684\u65f6\u7a7a\u7279\u5f81\uff0c\u589e\u5f3a\u6a21\u578b\u5bf9\u65f6\u7a7a\u8bed\u4e49\u7684\u7406\u89e3\u80fd\u529b\uff1b<\/li>\n<\/ul>\n<\/li>\n<li>\u7eaf\u5546\u6237\u8868\u5f81\uff08\u9488\u5bf9\u5b9e\u65f6\u70b9\u51fb\u5e8f\u5217\u3001\u5019\u9009\u5e8f\u5217\uff09\uff1a\u57fa\u4e8e\u5546\u6237+\u70ed\u9500\u54c1\u7684\u57fa\u7840\u56fe\u6587\u4fe1\u606f\uff0c\u8ba1\u7b97\u7eaf\u5546\u6237\u8868\u5f81\u3002<\/li>\n<\/ul>\n<p><strong>SFT+RL\u8bad\u7ec3\u6837\u672c\u6784\u5efa\uff1a<\/strong><\/p>\n<p>\u901a\u8fc7\u63a5\u5165vLLM\u63a8\u7406\u6846\u67b6\u4f18\u5316CoT\u5237\u6570\u6548\u7387\uff0c\u6784\u5efa100w+\u7684\u9ad8\u8d28\u91cfSFT\u51b7\u542f\u52a8\u6837\u672c\u3002\u9488\u5bf9SFT\u9636\u6bb5\u6570\u636e\u6b63\u4f8b\u5c11\u3001hitrate\u8fc7\u4f4e\u5bfc\u81f4\u7684RL\u9636\u6bb5reward\u7a00\u758f\u95ee\u9898\uff0c\u91c7\u7528\u5bf9\u5e97\u54c1\u63cf\u8ff0\u6570\u636e\u8fdb\u884c\u70ed\u9500\u54c1\u53bb\u91cd\u3001\u957f\u5ea6\u63a7\u5236\u3001\u4e0d\u89c4\u8303\u63cf\u8ff0\u7b5b\u9009\u7b49\u6570\u636e\u6e05\u6d17\u65b9\u6cd5\uff0c\u4ee5\u63d0\u5347\u6570\u636e\u8d28\u91cf\u3002<\/p>\n<p>\u5728\u6837\u672c\u6784\u5efa\u8fc7\u7a0b\u4e2d\uff0c\u901a\u8fc7\u6574\u5408\u5e97\u94fa\u3001\u7528\u6237\u57fa\u672c\u63a8\u8350\u53e3\u5f84\uff0c\u7cbe\u7b80\u590d\u8d2d\u8bb0\u5f55\u3001\u6e05\u6d17\u70b9\u51fb\u5e8f\u5217\u6570\u636e\uff0c\u6253\u901a\u8bad\u7ec3\u4e2d\u7684\u5546\u6237\u63cf\u8ff0\u67e5\u627e\u62fc\u63a5\u65b9\u5f0f\uff0c\u4f18\u5316\u51b7\u542f\u52a8SFT\u3001RL\u8bad\u7ec3\u6570\u636e\u6d41\u7a0b\uff0c\u5e76\u6784\u5efa\u63a8\u8350\u7279\u5f81\u65e5\u6eda\u8868\uff0c\u751f\u62101\u4ebf+RL\u8bad\u7ec3\u6837\u672c\u3002\u901a\u8fc7\u5efa\u7acb\u5b8c\u6574\u7684\u6570\u636epipeline\uff0c\u786e\u4fdd\u8bad\u7ec3\u6570\u636e\u7684\u65f6\u6548\u6027\u548c\u8d28\u91cf\u7a33\u5b9a\u6027\uff0c\u4e3a\u5927\u89c4\u6a21RL\u8bad\u7ec3\u63d0\u4f9b\u53ef\u9760\u7684\u6570\u636e\u652f\u6491\u3002<\/p>\n<h3>4.1.2 \u6a21\u578b\u67b6\u6784\u8bbe\u8ba1<\/h3>\n<p>\u5728\u4e0a\u8ff0\u7279\u5f81\u4f53\u7cfb\u548c\u6570\u636e\u6784\u5efa\u5de5\u4f5c\u7684\u57fa\u7840\u4e0a\uff0c\u6211\u4eec\u8bbe\u8ba1\u4e86\u591a\u6a21\u6001\u878d\u5408\u7684\u751f\u6210\u5f0franking\u67b6\u6784\u3002<\/p>\n<p>\u591a\u6a21\u6001\u878d\u5408\u67b6\u6784\uff1a \u53c2\u7167\u591a\u6a21\u6001\u5927\u6a21\u578b\uff08VLM\uff09\u7684\u8bbe\u8ba1\u601d\u8def\uff0c\u6211\u4eec\u6784\u5efa\u4e86\u5e97\u94fa\u878d\u5408\u8868\u5f81\u7f16\u7801\u5668\uff08GME + Projector\uff09\u4e0eLLM\u7684\u7ec4\u5408\u67b6\u6784\uff0c\u5b9e\u73b0\u591a\u6a21\u6001\u4fe1\u606f\u4e0e\u6587\u672c\u751f\u6210\u80fd\u529b\u7684\u6df1\u5ea6\u878d\u5408\u3002<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143524238.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p><strong>\u8f93\u5165\u8f93\u51fa\u8bbe\u8ba1**<\/strong>\uff1a**<\/p>\n<p>\u8f93\u5165\u4fe1\u606f\u5305\u62ec\uff1a<\/p>\n<p><strong>1. \u65f6\u7a7a\u4fe1\u606f**<\/strong>\uff1a**\u5f53\u524d\u65f6\u95f4\uff1a2025\u5e742\u670821\u65e5\u5468\u4e94\uff0c13:54:00\uff0c\u6211\u5728\u8fd1\u94c1\u5e7f\u573a\u70b9\u5916\u5356\uff1b<\/p>\n<p><strong>2. \u4e2a\u6027\u5316\u5386\u53f2**<\/strong>\uff1a**\u6211\u662f\u7537\u6027\uff0c18\u5c81\uff0c\u8fc7\u53bb\u4e00\u5e74\u70b9\u8fc7\u7684\u5916\u5356\u6709\uff1a<shop pad>...\uff1b<\/p>\n<p><strong>3. \u5b9e\u65f6\u5174\u8da3**<\/strong>\uff1a**\u6211\u521a\u70b9\u51fb\u8fc7 <shop pad>...\uff1b<\/p>\n<p><strong>4. \u5019\u9009\u96c6**<\/strong>\uff1a**\u4ece\u4ee5\u4e0b\u5e97\u94fa\u6311\u900920\u4e2a\u7528\u6237\u6700\u53ef\u80fd\u4e0b\u5355\u7684\u5e97\u94fa<special token1><shop pad><special token2><shop pad>,...\uff1b<\/p>\n<p>\u8f93\u51fa\u4fe1\u606f\u5305\u62ec\uff1a<\/p>\n<p>\u601d\u8003\u94fe+\u5019\u9009\u5e97\u94fa\u5e8f\u53f7\uff1a<think>...<\/think><special token355>...<special token209><\/p>\n<p><strong>\u6838\u5fc3\u5efa\u6a21\u6d41\u7a0b**<\/strong>\uff1a**<\/p>\n<p><strong>1. \u5e97\u94faToken\u9884\u7559\u673a\u5236**<\/strong>\uff1a**\u5728\u914d\u7f6e\u4e2d\u5b9a\u4e49\u7279\u6b8a\u7684shop_pad_token\uff08shop_token_index=151667\uff09\uff0c\u5728prompt\u6784\u5efa\u65f6\u5c06\u5e97\u94fa\u4f4d\u7f6e\u7528\u8be5\u7279\u6b8atoken\u586b\u5145\u3002\u4f8b\u5982\uff1a<\/p>\n<pre><code>person_history_text = f&quot;&quot;&quot;\u7528\u6237\u662f{data_dict[&quot;gender&quot;]}\u6027, {data_dict[&quot;user_type&quot;]}, \u5e74\u9f84{data_dict[&quot;age&quot;]}\u5c81, \u6700\u8fd1\u4e0b\u5355\u7684\u5e97\u94fa\u6709&lt;|shop_start|&gt;{recent_order_shop_id_pad_str}&lt;|shop_end|&gt;&quot;&quot;&quot;<\/code><\/pre>\n<p><strong>2. \u6587\u672c\u5d4c\u5165\u751f\u6210**<\/strong>\uff1a**\u5c06\u8bbe\u8ba1\u597d\u7684prompt\u8f93\u5165LLM\u7684embedding\u5c42\uff0c\u5f97\u5230\u5305\u542b\u9884\u7559\u4f4d\u7f6e\u7684\u6587\u672c\u5d4c\u5165\u5411\u91cf\uff1a<\/p>\n<pre><code>if inputs_embeds is None:    \n        inputs_embeds = self.get_input_embeddings()(input_ids)<\/code><\/pre>\n<p><strong>3. \u5e97\u94fa\u7279\u5f81\u6620\u5c04<\/strong>\uff1a\u901a\u8fc7\u5e97\u94fa\u878d\u5408\u8868\u5f81\u7f16\u7801\u5668\u83b7\u53d6\u5e97\u94fa\u7684\u591a\u6a21\u6001\u8868\u5f81\uff0c\u5e76\u901a\u8fc7projector\u6a21\u5757\u6620\u5c04\u5230\u4e0e\u6587\u672c\u5411\u91cf\u76f8\u540c\u7684\u7ef4\u5ea6\u7a7a\u95f4\uff1a<\/p>\n<pre><code>shop_features = None\nif shop_input_ids is not None:    \n    shop_features = self.get_shop_features(shop_input_ids=shop_input_ids)<\/code><\/pre>\n<p><strong>4. \u7279\u5f81\u66ff\u6362\u878d\u5408<\/strong>\uff1a\u6839\u636e\u914d\u7f6e\u7684shop_token_index\u751f\u6210mask\u5411\u91cf\uff0c\u6807\u8bc6prompt\u4e2d\u5e97\u94fatoken\u7684\u4f4d\u7f6e\uff0c\u7136\u540e\u7528\u6620\u5c04\u540e\u7684\u5e97\u94fa\u7279\u5f81\u66ff\u6362\u5bf9\u5e94\u4f4d\u7f6e\u7684\u6587\u672c\u5d4c\u5165\uff1a<\/p>\n<pre><code>special_shop_mask = (    \n    (input_ids == self.config.shop_token_index)    \n        .unsqueeze(-1)    .expand_as(inputs_embeds)   \n        .to(inputs_embeds.device)\n)\nshop_features = shop_features.to(inputs_embeds.device, inputs_embeds.dtype)\ninputs_embeds = inputs_embeds.masked_scatter(special_shop_mask, shop_features)<\/code><\/pre>\n<p><strong>5. \u7edf\u4e00\u751f\u6210\u5904\u7406<\/strong>\uff1a\u5c06\u66ff\u6362\u540e\u7684\u6574\u4f53embedding\u8f93\u5165LLM\u8fdb\u884c\u540e\u7eed\u7684\u751f\u6210\u5904\u7406\uff0c\u8f93\u51fathinking\u8fc7\u7a0b\u548c\u63a8\u8350\u7ed3\u679c\uff1a<\/p>\n<pre><code>outputs = self.language_model(    \n  attention_mask=attention_mask,   \n  position_ids=position_ids,   \n  inputs_embeds=inputs_embeds,   \n  # ... \u5176\u4ed6\u53c2\u6570\n)\nlogits = outputs[0]<\/code><\/pre>\n<p>``<\/p>\n<p><strong>\u67b6\u6784\u4f18\u52bf**<\/strong>\uff1a**<\/p>\n<p>\u8fd9\u79cd\u8bbe\u8ba1\u65e2\u4fdd\u6301\u4e86LLM\u5f3a\u5927\u7684\u6587\u672c\u751f\u6210\u80fd\u529b\uff0c\u53c8\u5145\u5206\u5229\u7528\u4e86\u524d\u671f\u591a\u6a21\u6001\u63a8\u8350\u4e2d\u79ef\u7d2f\u7684\u5e97\u94fa\u8868\u5f81\u80fd\u529b\u3002\u901a\u8fc7projector\u7684\u7ef4\u5ea6\u6620\u5c04\u548cmask\u673a\u5236\u7684\u7cbe\u786e\u66ff\u6362\uff0c\u5b9e\u73b0\u4e86\u7ed3\u6784\u5316\u5e97\u94fa\u4fe1\u606f\u4e0e\u81ea\u7136\u8bed\u8a00\u5904\u7406\u7684\u6709\u673a\u7ed3\u5408\u3002<\/p>\n<h3>4.1.3 \u8bad\u7ec3\u7b56\u7565<\/h3>\n<p>\u57fa\u4e8e\u4e0a\u8ff0\u6a21\u578b\u67b6\u6784\uff0c\u6211\u4eec\u8bbe\u8ba1\u4e86SFT+GRPO\u4e24\u9636\u6bb5\u8bad\u7ec3\u65b9\u6848\uff0c\u901a\u8fc7\u601d\u8003\u94fe\u8f93\u51fa\u6a21\u5f0f\u5f15\u5bfc\u6a21\u578b\u5728\u5f3a\u5316\u5b66\u4e60\u8fc7\u7a0b\u4e2d\u6d8c\u73b0\u63a8\u8350\u63a8\u7406\u80fd\u529b\u3002<\/p>\n<p><strong>\u76d1\u7763\u5fae\u8c03\u9636\u6bb5\uff08SFT\uff09**<\/strong>\uff1a**<\/p>\n<p>\u8bad\u7ec3\u67b6\u6784\uff1a\u5728\u4e0a\u8ff0\u5efa\u6a21\u65b9\u6cd5\u7684\u57fa\u7840\u4e0a\uff0c\u5c06\u6587\u672c\u6a21\u6001\u6570\u636e\uff08\u65f6\u7a7a\u4fe1\u606f+\u4e2a\u6027\u5316\u7279\u5f81+\u884c\u4e3a\u5e8f\u5217\uff09\u7684\u5d4c\u5165\u8868\u5f81\u4e0e\u7ecf\u8fc7\u5e97\u94fa\u7f16\u7801\u5668\u7684\u5e97\u94fa\u878d\u5408\u8868\u5f81\uff08\u5386\u53f2\u4e0b\u5355\u3001\u5b9e\u65f6\u70b9\u51fb\u3001\u5019\u9009750\u5e97\u94fa\uff09\u62fc\u63a5\u540e\u8f93\u5165LLM decoder\u8fdb\u884c\u76d1\u7763\u5fae\u8c03\uff0c\u8f93\u51fathinking\u8fc7\u7a0b\u548c\u63a8\u8350\u5546\u6237\u5e8f\u53f7\uff0c\u5b9e\u73b0\u5e97\u94fa\u878d\u5408\u8868\u5f81\u7f16\u7801\u5668\uff08GME + Projector\uff09\u4e0eLLM\u89e3\u7801\u5668\u7684\u8054\u5408\u8bad\u7ec3\u3002<\/p>\n<p>\u5173\u952e\u6280\u672f\u601d\u8def\uff1a\u5728thinking\u8fc7\u7a0b\u4e2d\u9700\u8981\u4f53\u73b0\u5546\u6237\u7684\u5173\u952e\u6587\u672c\u4fe1\u606f\uff0c\u901a\u8fc7\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u540c\u6b65\u66f4\u65b0GME\u6743\u91cd\u5b9e\u73b0\u3002\u5177\u4f53\u673a\u5236\u4e3a\uff1athinking\u8fc7\u7a0b\u7684token loss\u80fd\u591f\u4f20\u9012\u5230GME\u6a21\u5757\u66f4\u65b0\u6743\u91cd\uff0c\u4ece\u800c\u4f7f\u6a21\u578b\u5728thinking\u4e2d\u751f\u6210\u6b63\u786e\u7684\u5546\u6237\u63cf\u8ff0token\uff0c\u5b9e\u73b0\u591a\u6a21\u6001\u8868\u5f81\u4e0e\u6587\u672c\u751f\u6210\u7684\u6df1\u5ea6\u878d\u5408\u3002<\/p>\n<p>\u6838\u5fc3\u8bad\u7ec3\u76ee\u6807\uff1a<\/p>\n<p><strong>1. \u8bad\u7ec3thinking\u683c\u5f0f**<\/strong>\uff1a**\u4f7f\u6a21\u578b\u5b66\u4f1a\u6309\u7167\u9884\u5b9a\u683c\u5f0f\u8f93\u51fa\u7ed3\u6784\u5316\u7684\u601d\u8003\u8fc7\u7a0b\uff0c\u4e3a\u540e\u7eedRL\u9636\u6bb5\u7684\u903b\u8f91\u63a8\u7406\u80fd\u529b\u5960\u5b9a\u57fa\u7840<\/p>\n<p><strong>2. \u63d0\u5347hitrate**<\/strong>\uff1a**\u901a\u8fc7\u76d1\u7763\u5b66\u4e60\u63d0\u9ad8\u6a21\u578b\u5728\u63a8\u8350\u4efb\u52a1\u4e0a\u7684\u57fa\u7840\u547d\u4e2d\u7387\uff0c\u964d\u4f4eRL\u9636\u6bb5\u7684\u5956\u52b1\u7a00\u758f\u5ea6\u95ee\u9898\uff0c\u4e3a\u5f3a\u5316\u5b66\u4e60\u63d0\u4f9b\u66f4\u597d\u7684\u8d77\u70b9<\/p>\n<p><strong>\u5f3a\u5316\u5b66\u4e60\u9636\u6bb5\uff08GRPO\uff09**<\/strong>\uff1a**<\/p>\n<p>\u8bad\u7ec3\u7b56\u7565\uff1a\u5728SFT\u51b7\u542f\u52a8\u57fa\u7840\u4e0a\uff0c\u91c7\u7528GRPO\uff08Group Relative Policy Optimization\uff09\u7b97\u6cd5\u8fdb\u884c\u5f3a\u5316\u5b66\u4e60\u4f18\u5316\u3002\u901a\u8fc7\u8fed\u4ee3\u591a\u4e2a\u7248\u672c\u7684\u6570\u636e\uff0c\u4f7f\u7528\u7528\u6237\u6700\u7ec8\u4e0b\u5355\u884c\u4e3a\u4f5c\u4e3areward\u4fe1\u53f7\u8fdb\u884c\u6a21\u578b\u4f18\u5316\u3002<\/p>\n<p>\u80fd\u529b\u6d8c\u73b0\u673a\u5236\uff1aSFT\u9636\u6bb5\u4e3b\u8981\u786e\u4fdd\u8f93\u51fa\u683c\u5f0f\u7684\u89c4\u8303\u6027\u548c\u57fa\u7840\u63a8\u8350\u80fd\u529b\uff0c\u800c\u63a8\u8350\u903b\u8f91\u7684\u5408\u7406\u6027\u548c\u590d\u6742\u63a8\u7406\u80fd\u529b\u901a\u8fc7RL\u9636\u6bb5\u81ea\u4e3b\u6d8c\u73b0\u3002\u5728GRPO\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\uff0c\u6a21\u578b\u9010\u6e10\u5b66\u4f1a\u5728thinking\u8fc7\u7a0b\u4e2d\u4ea7\u751f\u66f4\u6df1\u5c42\u6b21\u7684\u53cd\u601d\u548c\u63a8\u7406\uff0c\u63d0\u5347\u63a8\u8350\u7ed3\u679c\u7684\u51c6\u786e\u6027\u548c\u53ef\u89e3\u91ca\u6027\u3002<\/p>\n<p>\u8bad\u7ec3\u6846\u67b6\uff1a\u57fa\u4e8e\u96c6\u56e2\u5185\u90e8\u7684ScaleAligner\u8bad\u7ec3\u6846\u67b6\uff08\u5df2\u5f00\u6e90\u4e3aROLL\uff09\u8fdb\u884c\u9002\u914d\uff0c\u4e3b\u8981\u5305\u62ec\u81ea\u5b9a\u4e49RL\u6570\u636e\u96c6\u9002\u914d\u3001\u6a21\u578b\u989d\u5916key\u7684\u4f20\u5165\u9002\u914d\uff08shop_input_ids, shop_attention_mask\uff09\u4ee5\u53ca\u81ea\u5b9a\u4e49reward\u9002\u914d\u7b49\u5173\u952e\u6a21\u5757\u7b49\u3002<\/p>\n<h3>4.1.4 \u6838\u5fc3\u6311\u6218\u4e0e\u53cd\u601d<\/h3>\n<p>\u867d\u7136\u4e0a\u8ff0\u65b9\u6848\u5728\u7406\u8bba\u8bbe\u8ba1\u548c\u6570\u636e\u6784\u5efa\u65b9\u9762\u8f83\u4e3a\u5b8c\u6574\uff0c\u4f46\u5728\u5b9e\u9645\u5b9e\u65bd\u8fc7\u7a0b\u4e2d\u9047\u5230\u4e86\u5173\u952e\u7684\u6280\u672f\u6311\u6218\u3002\u6211\u4eec\u5c06\u6a21\u578b\u5212\u5206\u4e3a\u5e97\u94fa\u8868\u5f81\u7f16\u7801\u5668\uff08GME\uff09\u548c\u63a8\u8350\u9884\u6d4bLLM\u89e3\u7801\u5668\u4e24\u4e2a\u72ec\u7acb\u6a21\u5757\uff0c\u4f46\u53d1\u73b0\u8be5\u5efa\u6a21\u65b9\u6848\u65e0\u6cd5\u5b9e\u73b0\u7aef\u5230\u7aef\u7684\u8054\u5408\u8bad\u7ec3\uff08cotrain\uff09\uff0c\u53ea\u80fd\u91c7\u7528\u5206\u6bb5\u5f0f\u8bad\u7ec3\u7b56\u7565\u3002\u6839\u672c\u539f\u56e0\u5728\u4e8e\u4e24\u4e2a\u6a21\u5757\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u5bf9\u8d44\u6e90\u7684\u9700\u6c42\u5b58\u5728\u663e\u8457\u5dee\u5f02\uff1a\u5e97\u94fa\u8868\u5f81\u7f16\u7801\u5668\u9700\u8981\u5927batch size\uff08\u901a\u5e38\u7b49\u4e8e\u6253\u5206\u6df1\u5ea6\uff0c\u5982750\u4e2a\u5019\u9009\u5e97\u94fa\uff09\uff0c\u800c\u63a8\u8350\u9884\u6d4b\u89e3\u7801\u5668\u9700\u8981\u5927\u6a21\u578b\u89c4\u6a21\uff0c\u4e24\u8005\u540c\u65f6\u8bad\u7ec3\u4f1a\u8d85\u51fa\u73b0\u6709\u786c\u4ef6\u8d44\u6e90\u9650\u5236\uff0c\u5b58\u5728\u6839\u672c\u6027\u7684\u6280\u672f\u77db\u76fe\u3002\u7531\u4e8e\u4e0a\u8ff0\u6280\u672f\u6311\u6218\uff0c\u8be5\u751f\u6210\u5f0f\u63a8\u8350\u65b9\u6848\u76ee\u524d\u5728\u5de5\u7a0b\u843d\u5730\u65b9\u9762\u5b58\u5728\u5361\u70b9\uff0c\u4f46\u8fd9\u4e00\u63a2\u7d22\u4e0d\u4ec5\u4e3a\u7406\u89e3\u751f\u6210\u5f0f\u63a8\u8350\u7684\u6280\u672f\u96be\u70b9\u548c\u5b9e\u73b0\u8def\u5f84\u63d0\u4f9b\u4e86\u5b9d\u8d35\u7ecf\u9a8c\uff0c\u5728\u7279\u5f81\u4f53\u7cfb\u8bbe\u8ba1\u548c\u8bad\u7ec3\u6570\u636e\u6784\u5efa\u65b9\u9762\u7684\u5de5\u4f5c\u4e5f\u4e3a\u540e\u7eed\u76f8\u5173\u9879\u76ee\u63d0\u4f9b\u4e86\u4e00\u5b9a\u7a0b\u5ea6\u7684\u590d\u7528\u4ef7\u503c\u3002<\/p>\n<p><strong>4.2 \u4e3b\u6d41\u7684\u751f\u6210\u5f0f\u63a8\u8350\u65b9\u6cd5\uff1a\u91cf\u5316\u7f16\u7801+LLM<\/strong><\/p>\n<p>25\u5e74\u4ee5\u6765\u4e1a\u754c\u5728\u751f\u6210\u5f0f\u63a8\u8350\u9886\u57df\u6d8c\u73b0\u51fa\u8bf8\u591a\u524d\u6cbf\u8fdb\u5c55\uff0c\u7279\u522b\u662f<strong>\u91cf\u5316\u7f16\u7801+LLM<\/strong>\u7684\u6280\u672f\u8def\u7ebf\u9010\u6e10\u6210\u4e3a\u4e3b\u6d41\uff0c\u6807\u5fd7\u7740\u63a8\u8350\u7cfb\u7edf\u6b63\u5411\u751f\u6210\u8303\u5f0f\u7684\u6f14\u8fdb\u65b9\u5411\u3002\u57fa\u4e8e\u524d\u8ff04.1\u8282\u5b9e\u8df5\u4e2d\u9047\u5230\u7684\u7aef\u5230\u7aef\u8bad\u7ec3\u6311\u6218\uff0c\u6211\u4eec\u53ef\u4ee5\u7406\u89e3\u4e3a\u4f55<strong>\u91cf\u5316\u7f16\u7801+LLM<\/strong>\u4f1a\u6210\u4e3a\u5f53\u524d\u751f\u6210\u5f0f\u63a8\u8350\u7684\u4e3b\u6d41\u6280\u672f\u8def\u7ebf\u3002\u8fd9\u79cd\u65b9\u6848\u901a\u8fc7\u5c06\u7528\u6237\u884c\u4e3a\u548c\u5546\u6237\u4fe1\u606f\u91cf\u5316\u4e3a\u79bb\u6563\u7684\u8bed\u4e49ID\uff0c\u7136\u540e\u5229\u7528LLM\u7684\u81ea\u56de\u5f52\u751f\u6210\u80fd\u529b\u8fdb\u884c\u63a8\u8350\uff0c\u4ece\u6839\u672c\u4e0a\u89e3\u51b3\u4e86\u591a\u6a21\u6001\u7f16\u7801\u5668\u4e0eLLM\u89e3\u7801\u5668\u8054\u5408\u8bad\u7ec3\u7684\u6280\u672f\u96be\u9898\u3002<\/p>\n<h3>4.2.1 \u6838\u5fc3\u6280\u672f\u601d\u8def<\/h3>\n<p><strong>\u91cf\u5316\u7f16\u7801\u7684\u4ef7\u503c**<\/strong>\uff1a<strong>\u91cf\u5316\u7f16\u7801\uff08\u4e3b\u8981\u57fa\u4e8eRQ-VAE\u7b49\u6280\u672f\uff09\u5c06\u8fde\u7eed\u7684\u591a\u6a21\u6001\u8868\u5f81\u8f6c\u6362\u4e3a\u79bb\u6563\u7684\u8bed\u4e49token\uff0c\u4f7f\u5f97\u4f20\u7edf\u63a8\u8350\u4fe1\u606f\u80fd\u591f\u88abLLM\u7406\u89e3\u548c\u5904\u7406\u3002\u8fd9\u79cd\u7f16\u7801\u65b9\u5f0f\u5177\u6709\u4e24\u4e2a\u5173\u952e\u4f18\u52bf\uff1a\u76f8\u4f3c\u5546\u54c1\u7684ID\u5728\u7f16\u7801\u7a7a\u95f4\u4e2d\u4e5f\u76f8\u8fd1\uff0c\u4fbf\u4e8e\u6cdb\u5316\u548c\u51b7\u542f\u52a8\uff1b\u5c42\u6b21\u5316\u7684\u7f16\u7801\u8fc7\u7a0b\u80fd\u591f\u4ece\u7c97\u5230\u7ec6\u5730\u611f\u77e5\u8bed\u4e49\u4fe1\u606f\u3002\u800c\u5728\u6211\u4eec\u7684\u751f\u6210\u5f0franking\u573a\u666f\u4e0b\uff0c\u5176\u66f4\u6838\u5fc3\u7684\u4ef7\u503c\u5728\u4e8e\uff1a<\/strong>\u7f29\u5c0f\u751f\u6210\u89e3\u7a7a\u95f4<strong>\uff0c\u5c06LLM\u9884\u6d4b\u8303\u56f4\u7f29\u5c0f\u81f3\u2264\u4e07\u7ea7\u522b\u7684\u8bed\u4e49ID\uff0c\u663e\u8457\u964d\u4f4eLLM\u5b66\u4e60\u96be\u5ea6\uff1b<\/strong>\u5bf9\u9f50item\u7279\u5f81\u4e0eLLM\u8bed\u4e49\u7a7a\u95f4**\uff0c\u4f7f\u5f97\u5546\u6237\u8868\u5f81\u80fd\u591f\u88abLLM\u76f4\u63a5\u7406\u89e3\u548c\u5904\u7406\uff0c\u907f\u514d\u4e86\u591a\u6a21\u6001\u7f16\u7801\u5668\u4e0eLLM\u89e3\u7801\u5668\u8054\u5408\u8bad\u7ec3\u7684\u8d44\u6e90\u51b2\u7a81\u3002<\/p>\n<p><strong>LLM\u751f\u6210\u8303\u5f0f**<\/strong>\uff1a**\u57fa\u4e8e\u7f16\u7801\u540e\u7684\u8bed\u4e49token\uff0c\u5229\u7528\u5927\u8bed\u8a00\u6a21\u578b\u7684\u81ea\u56de\u5f52\u751f\u6210\u80fd\u529b\u76f4\u63a5\u751f\u6210\u63a8\u8350\u7ed3\u679c\u3002\u8fd9\u79cd\u65b9\u5f0f\u4e0d\u4ec5\u7edf\u4e00\u4e86\u63a8\u8350\u6846\u67b6\uff0c\u8fd8\u5929\u7136\u5177\u5907\u53ef\u89e3\u91ca\u6027\u548c\u4ea4\u4e92\u80fd\u529b\uff0c\u80fd\u591f\u751f\u6210\u63a8\u8350\u7406\u7531\u548c\u652f\u6301\u591a\u8f6e\u5bf9\u8bdd\u3002<\/p>\n<p><strong>\u6280\u672f\u4f18\u52bf**<\/strong>\uff1a**\u76f8\u6bd4\u6211\u4eec\u63a2\u7d22\u7684\u591a\u6a21\u6001\u878d\u5408\u65b9\u6848\uff0c\u91cf\u5316\u7f16\u7801+LLM\u907f\u514d\u4e86\u4e0d\u540c\u6a21\u5757\u95f4\u7684\u8d44\u6e90\u9700\u6c42\u51b2\u7a81\uff0c\u53ef\u4ee5\u5b9e\u73b0\u7aef\u5230\u7aef\u8bad\u7ec3\uff1b\u540c\u65f6\u4fdd\u6301\u4e86\u5f3a\u5927\u7684\u6cdb\u5316\u80fd\u529b\u548c\u96f6\u6837\u672c\u63a8\u8350\u6f5c\u529b\uff0c\u5728\u53ec\u56de\u73af\u8282\u3001\u591a\u6a21\u6001\u7406\u89e3\u3001\u4f9b\u7ed9\u51b7\u542f\u52a8\u3001\u957f\u5c3e\u5174\u8da3\u6355\u6349\u7b49\u65b9\u9762\u5c55\u73b0\u51fa\u5de8\u5927\u6f5c\u529b\u3002<\/p>\n<p><strong>\u9002\u914d\u65b9\u6848\u601d\u8003\uff1a<\/strong>\u57fa\u4e8e\u91cf\u5316\u7f16\u7801\u6280\u672f\uff0c\u7b14\u8005\u5bf9\u4e8e4.1\u8282\u6240\u8ff0\u5efa\u6a21\u65b9\u6848\u7684\u4e00\u4e2a\u76f4\u89c2\u7684\u4f18\u5316\u601d\u8def\u662f\u53c2\u8003\u5de5\u4f5c[22]\u7684\u7f16\u7801\u65b9\u6848\uff0c\u4e00\u9636\u6bb5\u51bb\u7ed3\u5e97\u94fa\u878d\u5408\u8868\u5f81\u7f16\u7801\u5668\u6a21\u578b\u53c2\u6570\uff0c\u4ee5\u201c\u5c06\u5e97\u94fa\u8868\u5f81\u538b\u7f29\u6210 4 \u4e2a\u79bb\u6563\u7801\u5b57\uff0c\u540c\u65f6\u4fdd\u8bc1\u53ef\u91cd\u5efa\u201d\u4e3a\u8bad\u7ec3\u76ee\u6807\uff0c\u5bf9\u91cf\u5316\u7f16\u7801\u6a21\u5757\uff08DNN Encoder+RQ VAE+DNN Decoder\uff09\u8fdb\u884c\u79bb\u7ebf\u8bad\u7ec3\uff1b<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143524658.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<p>TIGER<\/p>\n<p>\u4e8c\u9636\u6bb5\u590d\u7528\u539f\u7248\u7279\u5f81\u4e0e\u6570\u636e\uff0c\u4fdd\u6301\u539f\u59cb\u5efa\u6a21\u4e0e\u8bad\u7ec3\u601d\u8def\u4e0d\u53d8\uff08\u5efa\u6a21\uff1aLLM Decoder\uff0c\u8f93\u5165\uff1a\u65f6\u7a7a\u4fe1\u606f+\u4e2a\u6027\u5316\u5386\u53f2+\u5b9e\u65f6\u5174\u8da3+\u5019\u9009\u96c6\uff0c\u8f93\u51fa\uff1a\u601d\u8003\u94fe+\u5019\u9009\u5e97\u94fa\uff0c\u8bad\u7ec3\uff1aSFT+GRPO\uff09\uff0c\u53ea\u9700\u8981\u5c06\u539f\u672c\u65b9\u6848\u4e2d\u8f93\u5165\u7684\u5e97\u94fa\u9884\u7559token\u4ee5\u53ca\u9884\u6d4b\u76ee\u6807\u4e2d\u7684\u5e97\u94fa\u5e8f\u53f7token\uff0c\u7528\u91cf\u5316\u7f16\u7801\u6a21\u5757\u4ea7\u51fa\u7684\u5e97\u94fa4\u4f4d\u7801\u5b57\u66ff\u4ee3\u5373\u53ef\uff0c\u5efa\u6a21\u601d\u8def\u5982\u4e0b\u56fe\uff1a<\/p>\n<p><img src=\"https:\/\/lemon-guess.oss-cn-hangzhou.aliyuncs.com\/img\/640-20251112143524901.png\" alt=\"\u56fe\u7247\" \/><\/p>\n<h3>4.2.2 \u4e1a\u754c\u524d\u6cbf\u8fdb\u5c55<\/h3>\n<p>\u5f53\u524d\u4e1a\u754c\u5728\u751f\u6210\u5f0f\u63a8\u8350\u9886\u57df\u6d8c\u73b0\u51fa\u8bf8\u591a\u521b\u65b0\u5de5\u4f5c\uff0c\u6db5\u76d6\u4e86\u91cf\u5316\u7f16\u7801\u4f18\u5316\u3001LLM\u4e0e\u63a8\u8350\u7cfb\u7edf\u878d\u5408\u3001\u591a\u6a21\u6001\u4fe1\u606f\u5904\u7406\u3001\u957f\u5e8f\u5217\u5efa\u6a21\u7b49\u591a\u4e2a\u6280\u672f\u65b9\u5411\u3002\u8fd9\u4e9b\u5de5\u4f5c\u5728\u8bed\u4e49ID\u6784\u5efa\u3001\u7279\u5f81\u5bf9\u9f50\u3001\u8bad\u7ec3\u7b56\u7565\u7b49\u65b9\u9762\u4e0d\u65ad\u7a81\u7834\uff0c\u63a8\u52a8\u7740\u751f\u6210\u5f0f\u63a8\u8350\u6280\u672f\u7684\u5feb\u901f\u53d1\u5c55\u3002<\/p>\n<p>\u7531\u4e8e\u751f\u6210\u5f0f\u63a8\u8350\u662f\u4e00\u4e2a\u5feb\u901f\u53d1\u5c55\u7684\u65b0\u5174\u9886\u57df\uff0c\u76f8\u5173\u6280\u672f\u7ec6\u8282\u548c\u6700\u65b0\u8fdb\u5c55\u5df2\u6709\u5f88\u591a\u4f18\u79c0\u6587\u7ae0\u8fdb\u884c\u4e86\u6df1\u5165\u89e3\u8bfb\uff0c\u672c\u6587\u4e0d\u518d\u4e00\u4e00\u5c55\u5f00\u3002<\/p>\n<p>\u7ed3\u8bed<\/p>\n<p>\u672c\u6587\u56de\u987e\u4e86\u7b14\u8005\u572824\u5e74\u4e0b\u534a\u5e74\u81f325\u5e74\u521d\u53c2\u4e0e\u591a\u6a21\u6001\u63a8\u8350\u548c\u751f\u6210\u5f0f\u63a8\u8350\u9879\u76ee\u4e2d\u7684\u5b66\u4e60\u5386\u7a0b\u4e0e\u6280\u672f\u5b9e\u8df5\u3002\u4ece\u4f20\u7edf\u63a8\u8350\u7b97\u6cd5\u5411\u591a\u6a21\u6001\u63a8\u8350\u7684\u6f14\u8fdb\uff0c\u5230\u591a\u6a21\u6001\u8868\u5f81\u6280\u672f\u7684\u6df1\u5165\u7406\u89e3\uff0c\u518d\u5230\u751f\u6210\u5f0f\u63a8\u8350\u7684\u524d\u6cbf\u63a2\u7d22\uff0c\u8fd9\u4e00\u8def\u8d70\u6765\u65e2\u6709\u6210\u529f\u7684\u5b9e\u8df5\u7ecf\u9a8c\uff0c\u4e5f\u6709\u672a\u80fd\u843d\u5730\u7684\u6280\u672f\u6311\u6218\uff0c\u6bcf\u4e00\u6b65\u90fd\u52a0\u6df1\u4e86\u5bf9\u63a8\u8350\u7cfb\u7edf\u6280\u672f\u53d1\u5c55\u8d8b\u52bf\u7684\u8ba4\u77e5\u3002\u9879\u76ee\u7684\u987a\u5229\u63a8\u8fdb\u79bb\u4e0d\u5f00\u5f88\u591a\u4f19\u4f34\u7684\u6089\u5fc3\u6307\u5bfc\u4e0e\u534f\u4f5c\u652f\u6301\uff0c\u5728\u6b64\u8868\u793a\u8bda\u631a\u7684\u611f\u8c22\u3002<\/p>\n<p>\u901a\u8fc7\u997f\u4e86\u4e48\u9996\u9875\u5e97\u94fa\u63a8\u8350\u573a\u666f\u7684\u591a\u6a21\u6001\u63a8\u8350\u7cfb\u7edf\u5b9e\u8df5\uff0c\u6211\u4eec\u9a8c\u8bc1\u4e86\u4e09\u9636\u6bb5\u8bad\u7ec3\u7b56\u7565\u7684\u6709\u6548\u6027\uff0c\u4e5f\u6df1\u523b\u4f53\u4f1a\u5230\u4e86\u591a\u6a21\u6001\u7279\u5f81\u4e0e\u4f20\u7edfID\u7279\u5f81\u878d\u5408\u7684\u590d\u6742\u6027\u3002\u800c\u5728\u751f\u6210\u5f0f\u63a8\u8350\u7684\u63a2\u7d22\u4e2d\uff0c\u867d\u7136\u9047\u5230\u4e86\u7aef\u5230\u7aef\u8bad\u7ec3\u7684\u6280\u672f\u74f6\u9888\uff0c\u4f46\u8fd9\u4e9b\u6311\u6218\u6070\u6070\u6307\u660e\u4e86\u672a\u6765\u6280\u672f\u53d1\u5c55\u7684\u65b9\u5411\u3002<\/p>\n<p>\u63a8\u8350\u7cfb\u7edf\u6b63\u7ad9\u5728\u4e00\u4e2a\u91cd\u8981\u7684\u6280\u672f\u8f6c\u6298\u70b9\u4e0a\u3002\u4ece\u534f\u540c\u8868\u5f81\u5230\u8054\u5408\u8868\u5f81\uff0c\u4ece\u5339\u914d\u8303\u5f0f\u5230\u751f\u6210\u8303\u5f0f\uff0c\u6280\u672f\u7684\u6bcf\u4e00\u6b21\u6f14\u8fdb\u90fd\u5728\u91cd\u65b0\u5b9a\u4e49\u7740\u63a8\u8350\u7cfb\u7edf\u7684\u80fd\u529b\u8fb9\u754c\u3002\u591a\u6a21\u6001\u63a8\u8350\u5df2\u7ecf\u5728\u5de5\u4e1a\u754c\u5f97\u5230\u5e7f\u6cdb\u5e94\u7528\uff0c\u800c\u751f\u6210\u5f0f\u63a8\u8350\u4f5c\u4e3a\u65b0\u5174\u6280\u672f\u65b9\u5411\uff0c\u6b63\u5728\u5feb\u901f\u53d1\u5c55\u5e76\u5c55\u73b0\u51fa\u5de8\u5927\u6f5c\u529b\u3002<\/p>\n<p>\u4f5c\u4e3a\u8fd9\u4e00\u6280\u672f\u53d8\u9769\u7684\u53c2\u4e0e\u8005\u548c\u89c1\u8bc1\u8005\uff0c\u7b14\u8005\u5c06\u6301\u7eed\u5173\u6ce8\u591a\u6a21\u6001\u63a8\u8350\u548c\u751f\u6210\u5f0f\u63a8\u8350\u6280\u672f\u7684\u53d1\u5c55\u52a8\u6001\uff0c\u5728\u5b9e\u8df5\u4e2d\u4e0d\u65ad\u5b66\u4e60\u548c\u63a2\u7d22\u3002\u5e0c\u671b\u8fd9\u4efd\u603b\u7ed3\u80fd\u591f\u4e3a\u540c\u6837\u5728\u8fd9\u4e00\u9886\u57df\u63a2\u7d22\u7684\u6280\u672f\u540c\u5b66\u63d0\u4f9b\u4e00\u4e9b\u53c2\u8003\uff0c\u4e5f\u671f\u5f85\u4e0e\u66f4\u591a\u540c\u5b66\u4ea4\u6d41\u5b66\u4e60\uff0c\u5171\u540c\u63a8\u52a8\u63a8\u8350\u7cfb\u7edf\u6280\u672f\u7684\u8fdb\u6b65\u3002<\/p>\n<h1><strong>Reference<\/strong><\/h1>\n<p>[1]Deep Interest Network for Click-Through Rate Prediction\uff1a<a href=\"https:\/\/arxiv.org\/abs\/1706.06978\">https:\/\/arxiv.org\/abs\/1706.06978<\/a><\/p>\n<p>[2]LightGCN: Simplifying and Powering Graph Convolution Network for Recommendation\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2002.02126\">https:\/\/arxiv.org\/abs\/2002.02126<\/a><\/p>\n<p>[3]Self-supervised Graph Learning for Recommendation\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2010.10783\">https:\/\/arxiv.org\/abs\/2010.10783<\/a><\/p>\n<p>[4]Contrastive Learning for Sequential Recommendation\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2010.14395\">https:\/\/arxiv.org\/abs\/2010.14395<\/a><\/p>\n<p>[5]Towards Universal Sequence Representation Learning for Recommender Systems\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2206.05941\">https:\/\/arxiv.org\/abs\/2206.05941<\/a><\/p>\n<p>[6]Recommendation as Language Processing (RLP): A Unified Pretrain, Personalized Prompt &amp; Predict Paradigm (P5)\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2203.13366\">https:\/\/arxiv.org\/abs\/2203.13366<\/a><\/p>\n<p>[7]AlignRec: Aligning and Training in Multimodal Recommendations\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/2403.12384\">https:\/\/arxiv.org\/pdf\/2403.12384<\/a><\/p>\n<p>[8]NoteLLM: A Retrievable Large Language Model for Note Recommendation\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/2403.01744\">https:\/\/arxiv.org\/pdf\/2403.01744<\/a><\/p>\n<p>[9]NoteLLM-2: Multimodal Large Representation Models for Recommendation<\/p>\n<p>[10]Enhancing Taobao Display Advertising with Multimodal Representations: Challenges, Approaches and Insights\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2405.16789\">https:\/\/arxiv.org\/abs\/2405.16789<\/a><\/p>\n<p>[11]QARM: Quantitative Alignment Multi-Modal Recommendation at Kuaishou\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/2411.11739\">https:\/\/arxiv.org\/pdf\/2411.11739<\/a><\/p>\n<p>[12]Towards General Text Embeddings with Multi-stage Contrastive Learning\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2308.03281\">https:\/\/arxiv.org\/abs\/2308.03281<\/a><\/p>\n<p>[13]LLM2Vec: Large Language Models Are Secretly Powerful Text Encoders\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2404.05961\">https:\/\/arxiv.org\/abs\/2404.05961<\/a><\/p>\n<p>[14]Qwen3 Embedding: Advancing Text Embedding and Reranking Through Foundation Models\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2506.05176\">https:\/\/arxiv.org\/abs\/2506.05176<\/a><\/p>\n<p>[15]Deep Residual Learning for Image Recognition\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/1512.03385\">https:\/\/arxiv.org\/pdf\/1512.03385<\/a><\/p>\n<p>[16]An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/2010.11929\">https:\/\/arxiv.org\/pdf\/2010.11929<\/a><\/p>\n<p>[17]Learning Transferable Visual Models From Natural Language Supervision\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2103.00020\">https:\/\/arxiv.org\/abs\/2103.00020<\/a><\/p>\n<p>[18]Align before Fuse: Vision and Language Representation Learning with Momentum Distillation\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2107.07651\">https:\/\/arxiv.org\/abs\/2107.07651<\/a><\/p>\n<p>[19]VLM2Vec: Training Vision-Language Models for Massive Multimodal Embedding Tasks\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2410.05160\">https:\/\/arxiv.org\/abs\/2410.05160<\/a><\/p>\n<p>[20]GME: Improving Universal Multimodal Retrieval by Multimodal LLMs\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2412.16855\">https:\/\/arxiv.org\/abs\/2412.16855<\/a><\/p>\n<p>[21]UniIR : Training and Benchmarking Universal Multimodal Information Retrievers\uff1a<a href=\"https:\/\/arxiv.org\/pdf\/2311.17136\">https:\/\/arxiv.org\/pdf\/2311.17136<\/a><\/p>\n<p>[22]Recommender Systems with Generative Retrieval\uff1a<a href=\"https:\/\/arxiv.org\/abs\/2305.05065\">https:\/\/arxiv.org\/abs\/2305.05065<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u5f15\u8a00 \u63a8\u8350\u7cfb\u7edf\u4f5c\u4e3a\u4e92\u8054\u7f51\u65f6\u4ee3\u8fde\u63a5\u7528\u6237\u4e0e\u4fe1\u606f\u7684\u6838\u5fc3\u6280\u672f\uff0c\u6b63\u5728\u7ecf\u5386\u4ece\u4f20\u7edf\u534f\u540c\u8fc7\u6ee4\u5411\u591a\u6a21\u6001\u667a\u80fd\u63a8\u8350\u7684\u91cd\u8981\u53d8\u9769\u3002\u968f\u7740\u6df1\u5ea6\u5b66\u4e60\u6280\u672f\u7684\u5feb\u901f\u53d1\u5c55\uff0c\u7279\u522b   \u2026 &#8230;<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"ngg_post_thumbnail":0},"categories":[290],"tags":[],"_links":{"self":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9506"}],"collection":[{"href":"\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=9506"}],"version-history":[{"count":1,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9506\/revisions"}],"predecessor-version":[{"id":9507,"href":"\/index.php?rest_route=\/wp\/v2\/posts\/9506\/revisions\/9507"}],"wp:attachment":[{"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=9506"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=9506"},{"taxonomy":"post_tag","embeddable":true,"href":"\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=9506"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}