mirror of
https://github.com/pese-git/llm-arch-research.git
synced 2026-01-23 13:00:54 +00:00
Sergey Penkovsky
148
assets/drawio/gpt1-architecture.drawio
Normal file
148
assets/drawio/gpt1-architecture.drawio
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
<mxfile host="65bd71144e">
|
||||||
|
<diagram name="GPT Architecture" id="DEYydPS-O6mnllJWumln">
|
||||||
|
<mxGraphModel dx="1216" dy="316" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||||
|
<root>
|
||||||
|
<mxCell id="0"/>
|
||||||
|
<mxCell id="1" parent="0"/>
|
||||||
|
<mxCell id="92" value="" style="group" vertex="1" connectable="0" parent="1">
|
||||||
|
<mxGeometry x="40" y="320" width="1286" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="3" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="230" width="440" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="4" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="51.42776556776556" y="50" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="22" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="350" y="80" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="5" value="Feed<div>Forward</div><div>Network</div>" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="260.9564102564102" y="50" width="71.9230769230769" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="7" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="379.997619047619" y="60" width="37.87142857142857" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="21" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="12" target="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="12" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="177.14285714285714" y="60" width="41.904761904761905" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="13" value="" style="endArrow=classic;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=elbowEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="3" target="4" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="20" y="80.00000000000011" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="14" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="18" target="12" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="155.71428571427464" y="79.99999999999989" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="18" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="150.00428571428571" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="19" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="4" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="213.80952380952382" y="410" as="sourcePoint"/>
|
||||||
|
<mxPoint x="145.71428571428578" y="80.00000000000011" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="23" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" target="24" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="236.85714285714286" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="349.7619047619048" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="236.67904761904765" y="125"/>
|
||||||
|
<mxPoint x="292.38095238095235" y="125"/>
|
||||||
|
<mxPoint x="355" y="125"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="28" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="24" target="7" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="24" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="350.00190476190477" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="25" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="34.325581395348834" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="150.71428571428578" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="34.25858250276859" y="130"/>
|
||||||
|
<mxPoint x="89.96048726467328" y="130"/>
|
||||||
|
<mxPoint x="155" y="130"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="36" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="32" target="3" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="32" value="+" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="90" width="110" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="33" value="Token Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="17.5" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="34" value="Position Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="100" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="46" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="37" target="40" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="37" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="690" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="38" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="7" target="37" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="47" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="40" target="44" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="40" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="790" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="49" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="41" target="42" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="41" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="950" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="52" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="42" target="50" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="42" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1050" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="48" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="44" target="41" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="44" value=".<div>.</div><div>.</div>" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="890" y="40" width="30" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="53" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="50" target="51" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="50" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1150" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="51" value="Softmax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1236" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="54" value="Tokens" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry y="40" width="60" height="90" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="55" style="edgeStyle=none;html=1;entryX=-0.025;entryY=0.538;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="54" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="42.75" y="84.66941747572821" as="sourcePoint"/>
|
||||||
|
<mxPoint x="90" y="85.33000000000004" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
</root>
|
||||||
|
</mxGraphModel>
|
||||||
|
</diagram>
|
||||||
|
</mxfile>
|
||||||
413
assets/drawio/gpt1-attention.drawio
Normal file
413
assets/drawio/gpt1-attention.drawio
Normal file
@@ -0,0 +1,413 @@
|
|||||||
|
<mxfile host="65bd71144e">
|
||||||
|
<diagram name="GPT Architecture" id="DEYydPS-O6mnllJWumln">
|
||||||
|
<mxGraphModel dx="2176" dy="702" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||||
|
<root>
|
||||||
|
<mxCell id="0"/>
|
||||||
|
<mxCell id="1" parent="0"/>
|
||||||
|
<mxCell id="92" value="" style="group" parent="1" vertex="1" connectable="0">
|
||||||
|
<mxGeometry x="40" y="320" width="1286" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="3" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;container=0;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="230" width="440" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="36" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="32" target="3" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="32" value="+" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="90" width="110" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="33" value="Token Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="17.5" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="34" value="Position Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="100" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="46" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="37" target="40" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="37" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="690" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="38" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="7" target="37" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="47" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="40" target="44" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="40" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="790" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="49" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="41" target="42" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="41" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="950" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="52" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="42" target="50" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="42" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1050" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="48" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="44" target="41" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="44" value=".<div>.</div><div>.</div>" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="890" y="40" width="30" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="53" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="50" target="51" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="50" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1150" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="51" value="Softmax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1236" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="54" value="Tokens" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry y="40" width="60" height="90" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="55" style="edgeStyle=none;html=1;entryX=-0.025;entryY=0.538;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="54" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="42.75" y="84.66941747572821" as="sourcePoint"/>
|
||||||
|
<mxPoint x="90" y="85.33000000000004" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="4" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF3333;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="281.42776556776556" y="50" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="22" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="580" y="80" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="5" value="Feed<div>Forward</div><div>Network</div>" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="490.9564102564102" y="50" width="71.9230769230769" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="7" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="609.9976190476191" y="60" width="37.87142857142857" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="21" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="12" target="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="12" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="407.1428571428571" y="60" width="41.904761904761905" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="13" value="" style="endArrow=classic;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=elbowEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="3" target="4" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="250" y="80.00000000000011" as="sourcePoint"/>
|
||||||
|
<mxPoint x="459.5238095238095" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="14" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="18" target="12" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="385.71428571427464" y="79.99999999999989" as="sourcePoint"/>
|
||||||
|
<mxPoint x="459.5238095238095" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="18" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="380.00428571428574" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="19" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="4" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="443.80952380952385" y="410" as="sourcePoint"/>
|
||||||
|
<mxPoint x="375.7142857142858" y="80.00000000000011" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="23" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" target="24" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="466.8571428571429" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="579.7619047619048" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="466.67904761904765" y="125"/>
|
||||||
|
<mxPoint x="522.3809523809523" y="125"/>
|
||||||
|
<mxPoint x="585" y="125"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="28" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="24" target="7" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="24" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="580.0019047619048" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="25" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="264.3255813953488" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="380.7142857142858" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="264.2585825027686" y="130"/>
|
||||||
|
<mxPoint x="319.96048726467325" y="130"/>
|
||||||
|
<mxPoint x="385" y="130"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="141" value="" style="endArrow=none;dashed=1;html=1;" edge="1" parent="92">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="180" y="250" as="sourcePoint"/>
|
||||||
|
<mxPoint x="281" y="110" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="142" value="" style="endArrow=none;dashed=1;html=1;entryX=1;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="1" target="4">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="620" y="560" as="sourcePoint"/>
|
||||||
|
<mxPoint x="660" y="520" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="218" value="" style="group;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" connectable="0" parent="1">
|
||||||
|
<mxGeometry x="130" y="660" width="680" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="195" style="edgeStyle=orthogonalEdgeStyle;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="143" target="147">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="196" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="143" target="148">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="197" style="edgeStyle=orthogonalEdgeStyle;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="143" target="149">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="143" value="X" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry y="60" width="40" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="147" value="W<sub>k</sub>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="80" width="40" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="199" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="148" target="151">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="148" value="W<sub>q</sub>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="80" y="60" width="40" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="149" value="W<sub>v</sub>" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="80" y="120" width="40" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="207" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0;entryY=1;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="150" target="158">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="236" y="20"/>
|
||||||
|
<mxPoint x="236" y="50"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="150" value="K" style="rounded=1;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="158.97000000000003" width="41.03" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="208" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="151" target="190">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="151" value="Q" style="rounded=1;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="158.97000000000003" y="60" width="41.03" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="214" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="152" target="187">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="600" y="140"/>
|
||||||
|
<mxPoint x="600" y="80"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="152" value="V" style="rounded=1;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="158.97000000000003" y="120" width="40" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="215" style="edgeStyle=none;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="187">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="680" y="80" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="187" value="O" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#fff2cc;strokeColor=#d6b656;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="620" y="60" width="40" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="211" style="edgeStyle=none;html=1;entryX=0;entryY=0;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="188" target="179">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="188" value="Scale" style="rounded=1;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;direction=east;rotation=90;fillColor=#f8cecc;strokeColor=#b85450;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="370" y="37.5" width="50" height="25" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="213" style="edgeStyle=orthogonalEdgeStyle;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="189" target="187">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="600" y="50"/>
|
||||||
|
<mxPoint x="600" y="80"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="189" value="Softmax" style="rounded=1;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;direction=east;rotation=90;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="218">
|
||||||
|
<mxGeometry x="530" y="37.5" width="50" height="25" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="209" style="edgeStyle=none;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="190" target="188">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="190" value="" style="group;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" connectable="0" parent="218">
|
||||||
|
<mxGeometry x="272.5" y="10" width="80" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="153" value="" style="whiteSpace=wrap;html=1;aspect=fixed;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry width="80" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="154" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="155" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="156" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="157" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="158" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="159" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="20" y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="160" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="40" y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="161" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="60" y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="162" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="163" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="20" y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="164" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="40" y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="165" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="60" y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="166" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="167" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="20" y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="168" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="40" y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="169" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="190">
|
||||||
|
<mxGeometry x="60" y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="191" value="" style="group;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" connectable="0" parent="218">
|
||||||
|
<mxGeometry x="440" y="10" width="80" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="170" value="" style="whiteSpace=wrap;html=1;aspect=fixed;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry width="80" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="171" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="172" value="" style="rounded=0;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="173" value="" style="rounded=0;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="174" value="" style="rounded=0;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="175" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="176" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="20" y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="177" value="" style="rounded=0;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="40" y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="178" value="" style="rounded=0;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="60" y="20" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="179" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="180" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="20" y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="181" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="40" y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="182" value="" style="rounded=0;whiteSpace=wrap;html=1;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="60" y="40" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="183" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="184" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="20" y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="185" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="40" y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="186" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#dae8fc;strokeColor=#6c8ebf;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" parent="191">
|
||||||
|
<mxGeometry x="60" y="60" width="20" height="20" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="198" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="147" target="150">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="200" style="edgeStyle=none;html=1;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" target="152">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="120" y="140" as="sourcePoint"/>
|
||||||
|
<mxPoint x="148.97000000000008" y="139.8599999999998" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="212" value="" style="endArrow=classic;html=1;exitX=1;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" edge="1" parent="218" source="182" target="189">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="610" y="50" as="sourcePoint"/>
|
||||||
|
<mxPoint x="660" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="219" value="" style="group;movable=0;resizable=0;rotatable=0;deletable=0;editable=0;locked=1;connectable=0;" vertex="1" connectable="0" parent="1">
|
||||||
|
<mxGeometry x="289.99776556776555" y="520" width="250.00223443223445" height="90" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="145" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" edge="1" parent="219" source="133" target="144">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="133" value="Concat" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;direction=east;rotation=90;" vertex="1" parent="219">
|
||||||
|
<mxGeometry x="132.50223443223445" y="32.5" width="50" height="25" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="136" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" edge="1" parent="219" target="133">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="108.97435897435912" y="45" as="sourcePoint"/>
|
||||||
|
<mxPoint x="250.00223443223445" y="25" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="129" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF3333;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" vertex="1" parent="219">
|
||||||
|
<mxGeometry x="30" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="130" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF3333;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" vertex="1" parent="219">
|
||||||
|
<mxGeometry x="20" y="10" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="131" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF3333;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" vertex="1" parent="219">
|
||||||
|
<mxGeometry x="10" y="20" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="132" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF3333;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" vertex="1" parent="219">
|
||||||
|
<mxGeometry y="30" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="146" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" edge="1" parent="219" source="144">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="250.00223443223445" y="44.969696969697" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="144" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;direction=east;rotation=90;" vertex="1" parent="219">
|
||||||
|
<mxGeometry x="182.50223443223445" y="32.5" width="50" height="25" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="220" value="" style="endArrow=none;dashed=1;html=1;" edge="1" parent="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="90" y="690" as="sourcePoint"/>
|
||||||
|
<mxPoint x="290" y="610" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="221" value="" style="endArrow=none;dashed=1;html=1;" edge="1" parent="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="370" y="610" as="sourcePoint"/>
|
||||||
|
<mxPoint x="830" y="700" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
</root>
|
||||||
|
</mxGraphModel>
|
||||||
|
</diagram>
|
||||||
|
</mxfile>
|
||||||
148
assets/drawio/gpt1-decoder.drawio
Normal file
148
assets/drawio/gpt1-decoder.drawio
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
<mxfile host="65bd71144e">
|
||||||
|
<diagram name="GPT Architecture" id="DEYydPS-O6mnllJWumln">
|
||||||
|
<mxGraphModel dx="979" dy="301" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||||
|
<root>
|
||||||
|
<mxCell id="0"/>
|
||||||
|
<mxCell id="1" parent="0"/>
|
||||||
|
<mxCell id="92" value="" style="group" parent="1" vertex="1" connectable="0">
|
||||||
|
<mxGeometry x="40" y="320" width="1286" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="3" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;strokeColor=#FF3333;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="230" width="440" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="4" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="51.42776556776556" y="50" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="22" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="350" y="80" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="5" value="Feed<div>Forward</div><div>Network</div>" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="260.9564102564102" y="50" width="71.9230769230769" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="7" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="379.997619047619" y="60" width="37.87142857142857" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="21" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="12" target="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="12" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="177.14285714285714" y="60" width="41.904761904761905" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="13" value="" style="endArrow=classic;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=elbowEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="3" target="4" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="20" y="80.00000000000011" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="14" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="18" target="12" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="155.71428571427464" y="79.99999999999989" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="18" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="150.00428571428571" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="19" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="4" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="213.80952380952382" y="410" as="sourcePoint"/>
|
||||||
|
<mxPoint x="145.71428571428578" y="80.00000000000011" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="23" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" target="24" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="236.85714285714286" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="349.7619047619048" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="236.67904761904765" y="125"/>
|
||||||
|
<mxPoint x="292.38095238095235" y="125"/>
|
||||||
|
<mxPoint x="355" y="125"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="28" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="24" target="7" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="24" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="350.00190476190477" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="25" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="34.325581395348834" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="150.71428571428578" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="34.25858250276859" y="130"/>
|
||||||
|
<mxPoint x="89.96048726467328" y="130"/>
|
||||||
|
<mxPoint x="155" y="130"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="36" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="32" target="3" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="32" value="+" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="90" width="110" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="33" value="Token Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="17.5" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="34" value="Position Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="100" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="46" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="37" target="40" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="37" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="690" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="38" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="7" target="37" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="47" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="40" target="44" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="40" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="790" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="49" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="41" target="42" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="41" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="950" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="52" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="42" target="50" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="42" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1050" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="48" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="44" target="41" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="44" value=".<div>.</div><div>.</div>" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="890" y="40" width="30" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="53" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="50" target="51" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="50" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1150" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="51" value="Softmax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1236" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="54" value="Tokens" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry y="40" width="60" height="90" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="55" style="edgeStyle=none;html=1;entryX=-0.025;entryY=0.538;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="54" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="42.75" y="84.66941747572821" as="sourcePoint"/>
|
||||||
|
<mxPoint x="90" y="85.33000000000004" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
</root>
|
||||||
|
</mxGraphModel>
|
||||||
|
</diagram>
|
||||||
|
</mxfile>
|
||||||
148
assets/drawio/gpt1-embeddings.drawio
Normal file
148
assets/drawio/gpt1-embeddings.drawio
Normal file
@@ -0,0 +1,148 @@
|
|||||||
|
<mxfile host="65bd71144e">
|
||||||
|
<diagram name="GPT Architecture" id="DEYydPS-O6mnllJWumln">
|
||||||
|
<mxGraphModel dx="1216" dy="316" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||||
|
<root>
|
||||||
|
<mxCell id="0"/>
|
||||||
|
<mxCell id="1" parent="0"/>
|
||||||
|
<mxCell id="91" value="" style="group;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="1" vertex="1" connectable="0">
|
||||||
|
<mxGeometry x="40" y="360" width="1286" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="56" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="230" width="440" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="57" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" vertex="1">
|
||||||
|
<mxGeometry x="51.42776556776556" y="50" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="58" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" source="59" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="350" y="80" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="59" value="Feed<div>Forward</div><div>Network</div>" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" vertex="1">
|
||||||
|
<mxGeometry x="260.9564102564102" y="50" width="71.9230769230769" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="60" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" vertex="1">
|
||||||
|
<mxGeometry x="379.997619047619" y="60" width="37.87142857142857" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="61" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" source="62" target="59" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="62" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" vertex="1">
|
||||||
|
<mxGeometry x="177.14285714285714" y="60" width="41.904761904761905" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="63" value="" style="endArrow=classic;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=elbowEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" source="56" target="57" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="20" y="80.00000000000011" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="64" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" source="65" target="62" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="155.71428571427464" y="79.99999999999989" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="65" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" vertex="1">
|
||||||
|
<mxGeometry x="150.00428571428571" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="66" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" source="57" target="65" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="213.80952380952382" y="410" as="sourcePoint"/>
|
||||||
|
<mxPoint x="145.71428571428578" y="80.00000000000011" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="67" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" target="69" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="236.85714285714286" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="349.7619047619048" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="236.67904761904765" y="125"/>
|
||||||
|
<mxPoint x="292.38095238095235" y="125"/>
|
||||||
|
<mxPoint x="355" y="125"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="68" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" source="69" target="60" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="69" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" vertex="1">
|
||||||
|
<mxGeometry x="350.00190476190477" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="70" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="56" target="65" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="34.325581395348834" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="150.71428571428578" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="34.25858250276859" y="130"/>
|
||||||
|
<mxPoint x="89.96048726467328" y="130"/>
|
||||||
|
<mxPoint x="155" y="130"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="71" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="72" target="56" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="72" value="+" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#FF3333;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="90" width="110" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="73" value="Token Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="95" y="17.5" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="74" value="Position Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="95" y="100" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="75" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="76" target="79" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="76" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="690" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="77" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="60" target="76" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="78" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="79" target="85" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="79" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="790" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="80" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="81" target="83" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="81" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="950" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="82" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="83" target="87" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="83" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="1050" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="84" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="85" target="81" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="85" value=".<div>.</div><div>.</div>" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="890" y="40" width="30" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="86" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="87" target="88" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="87" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="1150" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="88" value="Softmax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry x="1236" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="89" value="Tokens" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" vertex="1">
|
||||||
|
<mxGeometry y="40" width="60" height="90" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="90" style="edgeStyle=none;html=1;entryX=-0.025;entryY=0.538;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="91" source="89" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="42.75" y="84.66941747572821" as="sourcePoint"/>
|
||||||
|
<mxPoint x="90" y="85.33000000000004" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
</root>
|
||||||
|
</mxGraphModel>
|
||||||
|
</diagram>
|
||||||
|
</mxfile>
|
||||||
192
assets/drawio/gpt1-forward.drawio
Normal file
192
assets/drawio/gpt1-forward.drawio
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
<mxfile host="65bd71144e">
|
||||||
|
<diagram name="GPT Architecture" id="DEYydPS-O6mnllJWumln">
|
||||||
|
<mxGraphModel dx="2176" dy="1029" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
|
||||||
|
<root>
|
||||||
|
<mxCell id="0"/>
|
||||||
|
<mxCell id="1" parent="0"/>
|
||||||
|
<mxCell id="107" value="" style="group" vertex="1" connectable="0" parent="1">
|
||||||
|
<mxGeometry x="120" y="170" width="1286" height="265" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="92" value="" style="group" parent="107" vertex="1" connectable="0">
|
||||||
|
<mxGeometry width="1286" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="3" value="" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#fff2cc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="230" width="440" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="4" value="<div>Masked</div>Multi+Head<br>Attention" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="51.42776556776556" y="50" width="78.97435897435898" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="22" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="350" y="80" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="5" value="Feed<div>Forward</div><div>Network</div>" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF3333;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="260.9564102564102" y="50" width="71.9230769230769" height="60" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="7" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="379.997619047619" y="60" width="37.87142857142857" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="21" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="12" target="5" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="12" value="Norm" style="rounded=0;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="177.14285714285714" y="60" width="41.904761904761905" height="40" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="13" value="" style="endArrow=classic;html=1;exitX=0;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=elbowEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="3" target="4" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="20" y="80.00000000000011" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="14" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="18" target="12" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="155.71428571427464" y="79.99999999999989" as="sourcePoint"/>
|
||||||
|
<mxPoint x="229.52380952380952" y="-50" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="18" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="150.00428571428571" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="19" value="" style="endArrow=classic;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;edgeStyle=orthogonalEdgeStyle;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="4" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="213.80952380952382" y="410" as="sourcePoint"/>
|
||||||
|
<mxPoint x="145.71428571428578" y="80.00000000000011" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="23" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" target="24" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="236.85714285714286" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="349.7619047619048" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="236.67904761904765" y="125"/>
|
||||||
|
<mxPoint x="292.38095238095235" y="125"/>
|
||||||
|
<mxPoint x="355" y="125"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="28" value="" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" source="24" target="7" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="24" value="+" style="ellipse;whiteSpace=wrap;html=1;aspect=fixed;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" vertex="1">
|
||||||
|
<mxGeometry x="350.00190476190477" y="75" width="10" height="10" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="25" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="3" target="18" edge="1">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="34.325581395348834" y="80" as="sourcePoint"/>
|
||||||
|
<mxPoint x="150.71428571428578" y="85" as="targetPoint"/>
|
||||||
|
<Array as="points">
|
||||||
|
<mxPoint x="34.25858250276859" y="130"/>
|
||||||
|
<mxPoint x="89.96048726467328" y="130"/>
|
||||||
|
<mxPoint x="155" y="130"/>
|
||||||
|
</Array>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="104" value="" style="endArrow=none;dashed=1;html=1;" edge="1" parent="3">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="200" y="200" as="sourcePoint"/>
|
||||||
|
<mxPoint x="260.96000000000004" y="110" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="36" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="32" target="3" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="32" value="+" style="rounded=0;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="90" width="110" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="33" value="Token Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#6c8ebf;fillColor=#dae8fc;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="17.5" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="34" value="Position Emb" style="rounded=0;whiteSpace=wrap;html=1;strokeColor=#9673a6;fillColor=#e1d5e7;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="95" y="100" width="100" height="42.5" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="46" style="edgeStyle=none;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="37" target="40" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="37" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="690" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="38" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="7" target="37" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="47" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="40" target="44" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="40" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="790" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="49" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="41" target="42" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="41" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="950" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="52" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="42" target="50" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="42" value="Decoder" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#d5e8d4;strokeColor=#82b366;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1050" width="70" height="160" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="48" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="44" target="41" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="44" value=".<div>.</div><div>.</div>" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="890" y="40" width="30" height="80" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="53" style="edgeStyle=none;html=1;exitX=1;exitY=0.5;exitDx=0;exitDy=0;entryX=0;entryY=0.5;entryDx=0;entryDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="50" target="51" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="50" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1150" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="51" value="Softmax" style="rounded=1;whiteSpace=wrap;html=1;fillColor=#e1d5e7;strokeColor=#9673a6;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry x="1236" y="5" width="50" height="150" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="54" value="Tokens" style="rounded=1;whiteSpace=wrap;html=1;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" vertex="1">
|
||||||
|
<mxGeometry y="40" width="60" height="90" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="55" style="edgeStyle=none;html=1;entryX=-0.025;entryY=0.538;entryDx=0;entryDy=0;entryPerimeter=0;exitX=1;exitY=0.5;exitDx=0;exitDy=0;movable=1;resizable=1;rotatable=1;deletable=1;editable=1;locked=0;connectable=1;" parent="92" source="54" edge="1">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="42.75" y="84.66941747572821" as="sourcePoint"/>
|
||||||
|
<mxPoint x="90" y="85.33000000000004" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="105" value="" style="endArrow=none;dashed=1;html=1;exitX=1;exitY=1;exitDx=0;exitDy=0;" edge="1" parent="107" source="5">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint x="660" y="140" as="sourcePoint"/>
|
||||||
|
<mxPoint x="620" y="190" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="106" value="" style="group" vertex="1" connectable="0" parent="107">
|
||||||
|
<mxGeometry x="450" y="195" width="170" height="70" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="100" value="" style="edgeStyle=none;html=1;" edge="1" parent="106" source="93" target="99">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="93" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;rotation=90;container=0;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="106">
|
||||||
|
<mxGeometry x="-5" y="20" width="70" height="30" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="96" value="" style="endArrow=classic;html=1;entryX=0.5;entryY=1;entryDx=0;entryDy=0;" edge="1" parent="106" target="93">
|
||||||
|
<mxGeometry width="50" height="50" relative="1" as="geometry">
|
||||||
|
<mxPoint y="35" as="sourcePoint"/>
|
||||||
|
<mxPoint y="35.00999999999999" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="102" style="edgeStyle=none;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;" edge="1" parent="106" source="98">
|
||||||
|
<mxGeometry relative="1" as="geometry">
|
||||||
|
<mxPoint x="170" y="35.09433962264154" as="targetPoint"/>
|
||||||
|
</mxGeometry>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="98" value="Linear" style="rounded=1;whiteSpace=wrap;html=1;rotation=90;container=0;fillColor=#f5f5f5;fontColor=#333333;strokeColor=#666666;" vertex="1" parent="106">
|
||||||
|
<mxGeometry x="100" y="20" width="70" height="30" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="101" value="" style="edgeStyle=none;html=1;" edge="1" parent="106" source="99" target="98">
|
||||||
|
<mxGeometry relative="1" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
<mxCell id="99" value="ReLU" style="rounded=1;whiteSpace=wrap;html=1;rotation=90;container=0;fillColor=#e1d5e7;strokeColor=#9673a6;" vertex="1" parent="106">
|
||||||
|
<mxGeometry x="50" y="20" width="70" height="30" as="geometry"/>
|
||||||
|
</mxCell>
|
||||||
|
</root>
|
||||||
|
</mxGraphModel>
|
||||||
|
</diagram>
|
||||||
|
</mxfile>
|
||||||
BIN
assets/models/gpt1-architecture.png
Normal file
BIN
assets/models/gpt1-architecture.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 46 KiB |
BIN
assets/models/gpt1-attention.png
Normal file
BIN
assets/models/gpt1-attention.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 124 KiB |
BIN
assets/models/gpt1-decoder.png
Normal file
BIN
assets/models/gpt1-decoder.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 50 KiB |
BIN
assets/models/gpt1-embeddings.png
Normal file
BIN
assets/models/gpt1-embeddings.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 50 KiB |
BIN
assets/models/gpt1-forward.png
Normal file
BIN
assets/models/gpt1-forward.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 62 KiB |
@@ -4,7 +4,7 @@ from .feed_forward import FeedForward
|
|||||||
from .multi_head_attention import MultiHeadAttention
|
from .multi_head_attention import MultiHeadAttention
|
||||||
|
|
||||||
|
|
||||||
class Decoder(nn.Module):
|
class GptDecoder(nn.Module):
|
||||||
"""
|
"""
|
||||||
Decoder — базовый transformer decoder block (pre-LN), классический строительный блок современных языковых моделей.
|
Decoder — базовый transformer decoder block (pre-LN), классический строительный блок современных языковых моделей.
|
||||||
|
|
||||||
@@ -94,7 +94,13 @@ class Decoder(nn.Module):
|
|||||||
self._norm1 = nn.LayerNorm(emb_size)
|
self._norm1 = nn.LayerNorm(emb_size)
|
||||||
self._norm2 = nn.LayerNorm(emb_size)
|
self._norm2 = nn.LayerNorm(emb_size)
|
||||||
|
|
||||||
def forward(self, x: torch.Tensor, mask: torch.Tensor = None) -> torch.Tensor:
|
def forward(
|
||||||
|
self,
|
||||||
|
x: torch.Tensor,
|
||||||
|
use_cache: bool = False,
|
||||||
|
cache: list = None,
|
||||||
|
attention_mask=None
|
||||||
|
) -> tuple:
|
||||||
"""
|
"""
|
||||||
Один прямой проход через Transformer decoder block.
|
Один прямой проход через Transformer decoder block.
|
||||||
|
|
||||||
@@ -117,10 +123,16 @@ class Decoder(nn.Module):
|
|||||||
- Применяем FFN к нормализованному результату (layernorm)
|
- Применяем FFN к нормализованному результату (layernorm)
|
||||||
- Добавляем residual-связь (ffn + предыдущий выход)
|
- Добавляем residual-связь (ffn + предыдущий выход)
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Self-Attention блок
|
# Self-Attention блок
|
||||||
attention, _ = self._heads(x, mask, use_cache=False, cache=None)
|
attention, kv_caches = self._heads(x, attention_mask, use_cache=use_cache, cache=cache)
|
||||||
out = self._norm1(attention + x)
|
out = self._norm1(attention + x)
|
||||||
|
|
||||||
# FeedForward блок
|
# FeedForward блок
|
||||||
ffn_out = self._ff(out)
|
ffn_out = self._ff(out)
|
||||||
return self._norm2(ffn_out + out)
|
result = self._norm2(ffn_out + out)
|
||||||
|
|
||||||
|
if use_cache:
|
||||||
|
return (result, kv_caches)
|
||||||
|
else:
|
||||||
|
return (result, None)
|
||||||
@@ -209,14 +209,17 @@ class Gemma(BaseModel):
|
|||||||
else:
|
else:
|
||||||
return (logits, None)
|
return (logits, None)
|
||||||
|
|
||||||
def generate(self,
|
def generate(
|
||||||
x: torch.Tensor,
|
self,
|
||||||
max_new_tokens: int,
|
x: torch.Tensor,
|
||||||
|
max_new_tokens: int,
|
||||||
do_sample: bool,
|
do_sample: bool,
|
||||||
temperature: float = 1.0,
|
temperature: float = 1.0,
|
||||||
top_k: int = None,
|
top_k: int = None,
|
||||||
top_p: float = None,
|
top_p: float = None,
|
||||||
use_cache: bool = True
|
use_cache: bool = True,
|
||||||
|
attention_mask: torch.Tensor = None,
|
||||||
|
**kwargs
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Авторегрессивная генерация токенов с использованием greedy, temperature, top-k и top-p sampling.
|
Авторегрессивная генерация токенов с использованием greedy, temperature, top-k и top-p sampling.
|
||||||
|
|||||||
@@ -26,7 +26,7 @@ import torch.nn as nn
|
|||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from typing import Optional, Dict
|
from typing import Optional, Dict
|
||||||
from llm.core.base_model import BaseModel
|
from llm.core.base_model import BaseModel
|
||||||
from llm.core.decoder import Decoder
|
from llm.core.gpt_decoder import GptDecoder
|
||||||
from llm.core.token_embeddings import TokenEmbeddings
|
from llm.core.token_embeddings import TokenEmbeddings
|
||||||
from llm.core.positional_embeddings import PositionalEmbeddings
|
from llm.core.positional_embeddings import PositionalEmbeddings
|
||||||
|
|
||||||
@@ -116,7 +116,7 @@ class GPT(BaseModel):
|
|||||||
# head_size = emb_size // num_heads
|
# head_size = emb_size // num_heads
|
||||||
self._decoders = nn.ModuleList(
|
self._decoders = nn.ModuleList(
|
||||||
[
|
[
|
||||||
Decoder(
|
GptDecoder(
|
||||||
num_heads=config["num_heads"],
|
num_heads=config["num_heads"],
|
||||||
emb_size=config["embed_dim"],
|
emb_size=config["embed_dim"],
|
||||||
head_size=config["embed_dim"] // config["num_heads"],
|
head_size=config["embed_dim"] // config["num_heads"],
|
||||||
@@ -133,7 +133,9 @@ class GPT(BaseModel):
|
|||||||
"""Возвращает максимальную длину последовательности."""
|
"""Возвращает максимальную длину последовательности."""
|
||||||
return self._max_seq_len
|
return self._max_seq_len
|
||||||
|
|
||||||
def forward(self, x: torch.Tensor, attention_mask=None) -> torch.Tensor:
|
def forward(
|
||||||
|
self, x: torch.Tensor, attention_mask=None, use_cache: bool = True, cache: list = None
|
||||||
|
) -> tuple:
|
||||||
"""
|
"""
|
||||||
Прямой проход для получения логитов по последовательности токенов.
|
Прямой проход для получения логитов по последовательности токенов.
|
||||||
|
|
||||||
@@ -157,33 +159,60 @@ class GPT(BaseModel):
|
|||||||
f"Длина последовательности {x.size(1)} превышает максимальную {self._max_seq_len}"
|
f"Длина последовательности {x.size(1)} превышает максимальную {self._max_seq_len}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Вычисление start_pos из кэша (если кэш передан)
|
||||||
|
if cache is not None:
|
||||||
|
seq_len = 1
|
||||||
|
# Безопасно извлекаем key_cache для вычисления start_pos
|
||||||
|
if (
|
||||||
|
isinstance(cache, (list, tuple))
|
||||||
|
and len(cache) > 0
|
||||||
|
and cache[0] is not None
|
||||||
|
and isinstance(cache[0], (list, tuple))
|
||||||
|
and len(cache[0]) > 0
|
||||||
|
and cache[0][0] is not None
|
||||||
|
and isinstance(cache[0][0], (tuple, list))
|
||||||
|
and len(cache[0][0]) > 0
|
||||||
|
):
|
||||||
|
key_cache, _ = cache[0][0]
|
||||||
|
start_pos = key_cache.size(1)
|
||||||
|
else:
|
||||||
|
start_pos = 0
|
||||||
|
else:
|
||||||
|
# Без кэша работаем как раньше
|
||||||
|
start_pos = 0
|
||||||
|
seq_len = x.size(1)
|
||||||
|
|
||||||
# Эмбеддинги токенов и позиций
|
# Эмбеддинги токенов и позиций
|
||||||
tok_out = self._token_embeddings(x) # [batch, seq_len, emb_size]
|
tok_out = self._token_embeddings(x) # [batch, seq_len, emb_size]
|
||||||
pos_out = self._position_embeddings(x.size(1)) # [seq_len, emb_size]
|
pos_out = self._position_embeddings(
|
||||||
|
seq_len, start_pos=start_pos
|
||||||
|
) # [seq_len, emb_size]
|
||||||
|
|
||||||
# Комбинирование
|
# Комбинирование
|
||||||
out = self._dropout(
|
out = self._dropout(
|
||||||
tok_out + pos_out.unsqueeze(0)
|
tok_out + pos_out.unsqueeze(0)
|
||||||
) # [batch, seq_len, emb_size]
|
) # [batch, seq_len, emb_size]
|
||||||
|
|
||||||
# Стек декодеров
|
# Стек декодеров с передачей кэша
|
||||||
for decoder in self._decoders:
|
new_cache = []
|
||||||
out = decoder(out)
|
for i, decoder in enumerate(self._decoders):
|
||||||
|
decoder_cache = cache[i] if cache is not None else None
|
||||||
|
decoder_result = decoder(out, use_cache=use_cache, cache=decoder_cache)
|
||||||
|
|
||||||
return self._linear(out) # [batch, seq_len, vocab_size]
|
# Извлекаем результат из кортежа
|
||||||
|
if use_cache:
|
||||||
|
out, decoder_new_cache = decoder_result
|
||||||
|
new_cache.append(decoder_new_cache)
|
||||||
|
else:
|
||||||
|
out = decoder_result[0]
|
||||||
|
|
||||||
# def forward(self, input_ids, attention_mask=None):
|
logits = self._linear(out) # [batch, seq_len, vocab_size]
|
||||||
# B, T = input_ids.size()
|
|
||||||
# pos = torch.arange(0, T, device=input_ids.device).unsqueeze(0)
|
# Возвращаем результат с учетом use_cache
|
||||||
#
|
if use_cache:
|
||||||
# x = self.token_emb(input_ids) + self.pos_emb(pos)
|
return (logits, new_cache)
|
||||||
#
|
else:
|
||||||
# for block in self.blocks:
|
return (logits, None)
|
||||||
# x = block(x, attention_mask)
|
|
||||||
#
|
|
||||||
# x = self.ln_f(x)
|
|
||||||
# logits = self.head(x)
|
|
||||||
# return logits
|
|
||||||
|
|
||||||
def generate(
|
def generate(
|
||||||
self,
|
self,
|
||||||
@@ -193,8 +222,9 @@ class GPT(BaseModel):
|
|||||||
temperature: float = 1.0,
|
temperature: float = 1.0,
|
||||||
top_k: int = None,
|
top_k: int = None,
|
||||||
top_p: float = None,
|
top_p: float = None,
|
||||||
attention_mask: torch.Tensor = None, # Добавляем для совместимости с HF
|
use_cache: bool = True,
|
||||||
**kwargs, # Игнорируем остальные параметры
|
attention_mask: torch.Tensor = None,
|
||||||
|
**kwargs
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Авторегрессивная генерация текста с поддержкой жадного поиска (greedy), вероятностного сэмплирования с температурой,
|
Авторегрессивная генерация текста с поддержкой жадного поиска (greedy), вероятностного сэмплирования с температурой,
|
||||||
@@ -244,12 +274,24 @@ class GPT(BaseModel):
|
|||||||
- Holtzman et al., "The Curious Case of Neural Text Degeneration" (nucleus sampling): https://arxiv.org/abs/1904.09751
|
- Holtzman et al., "The Curious Case of Neural Text Degeneration" (nucleus sampling): https://arxiv.org/abs/1904.09751
|
||||||
- Оригинальный GPT-2: https://cdn.openai.com/better-language-models/language-models.pdf
|
- Оригинальный GPT-2: https://cdn.openai.com/better-language-models/language-models.pdf
|
||||||
"""
|
"""
|
||||||
|
cache = None
|
||||||
|
|
||||||
for _ in range(max_new_tokens):
|
for _ in range(max_new_tokens):
|
||||||
# 1. Обрезаем вход, если последовательность слишком длинная
|
# 1. Обрезаем вход, если последовательность слишком длинная
|
||||||
x_cond = x[:, -self._max_seq_len :]
|
if use_cache and cache is not None:
|
||||||
|
# Используем кэш - передаем только последний токен
|
||||||
|
x_input = x[:, -1:] # [batch_size, 1]
|
||||||
|
else:
|
||||||
|
# Первая итерация или кэш отключен - передаем всю последовательность
|
||||||
|
x_input = x
|
||||||
|
|
||||||
# 2. Передаем последовательность в метод forward класса GPT и полуаем логиты.
|
# 2. Передаем последовательность в метод forward класса GPT и полуаем логиты.
|
||||||
logits = self.forward(x_cond)
|
# Прямой проход с кэшем
|
||||||
|
logits, new_cache = self.forward(x_input, use_cache=use_cache, cache=cache)
|
||||||
|
|
||||||
|
# Обновляем кэш для следующей итерации
|
||||||
|
if use_cache:
|
||||||
|
cache = new_cache
|
||||||
|
|
||||||
# 3. Берем логиты для последнего токена
|
# 3. Берем логиты для последнего токена
|
||||||
last_logits = logits[:, -1, :] # [batch_size, vocab_size]
|
last_logits = logits[:, -1, :] # [batch_size, vocab_size]
|
||||||
|
|||||||
@@ -214,6 +214,8 @@ class GPT2(BaseModel):
|
|||||||
top_k: int = None,
|
top_k: int = None,
|
||||||
top_p: float = None,
|
top_p: float = None,
|
||||||
use_cache: bool = True,
|
use_cache: bool = True,
|
||||||
|
attention_mask: torch.Tensor = None,
|
||||||
|
**kwargs
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Авторегрессивная генерация токенов с поддержкой greedy, temperature, top-k, top-p sampling и KV-кэша.
|
Авторегрессивная генерация токенов с поддержкой greedy, temperature, top-k, top-p sampling и KV-кэша.
|
||||||
|
|||||||
@@ -176,6 +176,8 @@ class Llama(BaseModel):
|
|||||||
top_k: int = None,
|
top_k: int = None,
|
||||||
top_p: float = None,
|
top_p: float = None,
|
||||||
use_cache: bool = True,
|
use_cache: bool = True,
|
||||||
|
attention_mask: torch.Tensor = None,
|
||||||
|
**kwargs
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Авторегрессивная генерация последовательностей на основе LLaMA (greedy, temperature, top-k, top-p/nucleus, поддержка KV-кэша).
|
Авторегрессивная генерация последовательностей на основе LLaMA (greedy, temperature, top-k, top-p/nucleus, поддержка KV-кэша).
|
||||||
|
|||||||
@@ -140,14 +140,17 @@ class Mistral(BaseModel):
|
|||||||
else:
|
else:
|
||||||
return (logits, None)
|
return (logits, None)
|
||||||
|
|
||||||
def generate(self,
|
def generate(
|
||||||
x: torch.Tensor,
|
self,
|
||||||
max_new_tokens: int,
|
x: torch.Tensor,
|
||||||
|
max_new_tokens: int,
|
||||||
do_sample: bool,
|
do_sample: bool,
|
||||||
temperature: float = 1.0,
|
temperature: float = 1.0,
|
||||||
top_k: int = None,
|
top_k: int = None,
|
||||||
top_p: float = None,
|
top_p: float = None,
|
||||||
use_cache: bool = True
|
use_cache: bool = True,
|
||||||
|
attention_mask: torch.Tensor = None,
|
||||||
|
**kwargs
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Авторегрессивная генерация токенов с поддержкой greedy, temperature, top-k/top-p sampling
|
Авторегрессивная генерация токенов с поддержкой greedy, temperature, top-k/top-p sampling
|
||||||
|
|||||||
@@ -222,14 +222,17 @@ class Mixtral(BaseModel):
|
|||||||
else:
|
else:
|
||||||
return (logits, None)
|
return (logits, None)
|
||||||
|
|
||||||
def generate(self,
|
def generate(
|
||||||
x: torch.Tensor,
|
self,
|
||||||
max_new_tokens: int,
|
x: torch.Tensor,
|
||||||
|
max_new_tokens: int,
|
||||||
do_sample: bool,
|
do_sample: bool,
|
||||||
temperature: float = 1.0,
|
temperature: float = 1.0,
|
||||||
top_k: int = None,
|
top_k: int = None,
|
||||||
top_p: float = None,
|
top_p: float = None,
|
||||||
use_cache: bool = True
|
use_cache: bool = True,
|
||||||
|
attention_mask: torch.Tensor = None,
|
||||||
|
**kwargs
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
Авторегрессивная генерация токенов с поддержкой greedy, temperature, top-k/top-p sampling
|
Авторегрессивная генерация токенов с поддержкой greedy, temperature, top-k/top-p sampling
|
||||||
|
|||||||
@@ -4,17 +4,17 @@ Tests for decoder block.
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import torch
|
import torch
|
||||||
from llm.core.decoder import Decoder
|
from llm.core.gpt_decoder import GptDecoder
|
||||||
|
|
||||||
|
|
||||||
class TestDecoder:
|
class TestGptDecoder:
|
||||||
"""Test cases for Decoder."""
|
"""Test cases for Decoder."""
|
||||||
|
|
||||||
def test_initialization(self, embed_dim, num_heads):
|
def test_initialization(self, embed_dim, num_heads):
|
||||||
"""Test that Decoder can be initialized."""
|
"""Test that Decoder can be initialized."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -32,7 +32,7 @@ class TestDecoder:
|
|||||||
"""Test forward pass of Decoder."""
|
"""Test forward pass of Decoder."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -40,7 +40,7 @@ class TestDecoder:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Forward pass
|
# Forward pass
|
||||||
output = decoder(random_embeddings)
|
output, _ = decoder(random_embeddings)
|
||||||
|
|
||||||
# Check output shape
|
# Check output shape
|
||||||
assert output.shape == random_embeddings.shape
|
assert output.shape == random_embeddings.shape
|
||||||
@@ -50,7 +50,7 @@ class TestDecoder:
|
|||||||
"""Test forward pass with causal mask."""
|
"""Test forward pass with causal mask."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -62,7 +62,7 @@ class TestDecoder:
|
|||||||
mask = torch.tril(torch.ones(seq_len, seq_len))
|
mask = torch.tril(torch.ones(seq_len, seq_len))
|
||||||
|
|
||||||
# Forward pass with causal mask
|
# Forward pass with causal mask
|
||||||
output = decoder(random_embeddings, mask=mask)
|
output, _ = decoder(random_embeddings, attention_mask=mask)
|
||||||
|
|
||||||
# Check output shape
|
# Check output shape
|
||||||
assert output.shape == random_embeddings.shape
|
assert output.shape == random_embeddings.shape
|
||||||
@@ -71,14 +71,14 @@ class TestDecoder:
|
|||||||
"""Test that residual connections are properly applied."""
|
"""Test that residual connections are properly applied."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
max_seq_len=max_seq_len,
|
max_seq_len=max_seq_len,
|
||||||
)
|
)
|
||||||
|
|
||||||
output = decoder(random_embeddings)
|
output, _ = decoder(random_embeddings)
|
||||||
|
|
||||||
# With residual connections and layer norm, the output shouldn't be
|
# With residual connections and layer norm, the output shouldn't be
|
||||||
# too different from input (in terms of scale/distribution)
|
# too different from input (in terms of scale/distribution)
|
||||||
@@ -92,14 +92,14 @@ class TestDecoder:
|
|||||||
"""Test that layer normalization is applied."""
|
"""Test that layer normalization is applied."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
max_seq_len=max_seq_len,
|
max_seq_len=max_seq_len,
|
||||||
)
|
)
|
||||||
|
|
||||||
output = decoder(random_embeddings)
|
output, _ = decoder(random_embeddings)
|
||||||
|
|
||||||
# Check that output has reasonable statistics (due to layer norm)
|
# Check that output has reasonable statistics (due to layer norm)
|
||||||
# Mean should be close to 0, std close to 1 for each sequence position
|
# Mean should be close to 0, std close to 1 for each sequence position
|
||||||
@@ -114,7 +114,7 @@ class TestDecoder:
|
|||||||
"""Test that gradients flow through Decoder."""
|
"""Test that gradients flow through Decoder."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -122,7 +122,7 @@ class TestDecoder:
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Forward pass
|
# Forward pass
|
||||||
output = decoder(random_embeddings)
|
output, _ = decoder(random_embeddings)
|
||||||
|
|
||||||
# Create a dummy loss and backward pass
|
# Create a dummy loss and backward pass
|
||||||
loss = output.sum()
|
loss = output.sum()
|
||||||
@@ -139,7 +139,7 @@ class TestDecoder:
|
|||||||
"""Test that Decoder works on correct device."""
|
"""Test that Decoder works on correct device."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -148,7 +148,7 @@ class TestDecoder:
|
|||||||
inputs = random_embeddings.to(device)
|
inputs = random_embeddings.to(device)
|
||||||
|
|
||||||
# Forward pass
|
# Forward pass
|
||||||
output = decoder(inputs)
|
output, _ = decoder(inputs)
|
||||||
|
|
||||||
# Check device consistency
|
# Check device consistency
|
||||||
assert output.device == device
|
assert output.device == device
|
||||||
@@ -165,7 +165,7 @@ class TestDecoder:
|
|||||||
for embed_dim, num_heads in test_cases:
|
for embed_dim, num_heads in test_cases:
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -174,7 +174,7 @@ class TestDecoder:
|
|||||||
batch_size, seq_len = 2, 16
|
batch_size, seq_len = 2, 16
|
||||||
inputs = torch.randn(batch_size, seq_len, embed_dim)
|
inputs = torch.randn(batch_size, seq_len, embed_dim)
|
||||||
|
|
||||||
output = decoder(inputs)
|
output, _ = decoder(inputs)
|
||||||
|
|
||||||
assert output.shape == inputs.shape
|
assert output.shape == inputs.shape
|
||||||
|
|
||||||
@@ -183,7 +183,7 @@ class TestDecoder:
|
|||||||
"""Test Decoder with different input shapes."""
|
"""Test Decoder with different input shapes."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -191,7 +191,7 @@ class TestDecoder:
|
|||||||
)
|
)
|
||||||
|
|
||||||
inputs = torch.randn(batch_size, seq_len, embed_dim)
|
inputs = torch.randn(batch_size, seq_len, embed_dim)
|
||||||
output = decoder(inputs)
|
output, _ = decoder(inputs)
|
||||||
|
|
||||||
assert output.shape == (batch_size, seq_len, embed_dim)
|
assert output.shape == (batch_size, seq_len, embed_dim)
|
||||||
|
|
||||||
@@ -199,7 +199,7 @@ class TestDecoder:
|
|||||||
"""Test that Decoder behaves differently in train vs eval mode."""
|
"""Test that Decoder behaves differently in train vs eval mode."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -209,11 +209,11 @@ class TestDecoder:
|
|||||||
|
|
||||||
# Training mode
|
# Training mode
|
||||||
decoder.train()
|
decoder.train()
|
||||||
output_train = decoder(random_embeddings)
|
output_train, _ = decoder(random_embeddings)
|
||||||
|
|
||||||
# Evaluation mode
|
# Evaluation mode
|
||||||
decoder.eval()
|
decoder.eval()
|
||||||
output_eval = decoder(random_embeddings)
|
output_eval, _ = decoder(random_embeddings)
|
||||||
|
|
||||||
# Outputs should be different due to dropout
|
# Outputs should be different due to dropout
|
||||||
assert not torch.allclose(output_train, output_eval)
|
assert not torch.allclose(output_train, output_eval)
|
||||||
@@ -222,7 +222,7 @@ class TestDecoder:
|
|||||||
"""Test that parameters are properly initialized."""
|
"""Test that parameters are properly initialized."""
|
||||||
head_size = embed_dim // num_heads
|
head_size = embed_dim // num_heads
|
||||||
max_seq_len = 1024
|
max_seq_len = 1024
|
||||||
decoder = Decoder(
|
decoder = GptDecoder(
|
||||||
num_heads=num_heads,
|
num_heads=num_heads,
|
||||||
emb_size=embed_dim,
|
emb_size=embed_dim,
|
||||||
head_size=head_size,
|
head_size=head_size,
|
||||||
@@ -30,7 +30,7 @@ class TestGPT:
|
|||||||
model = GPT(gpt_config)
|
model = GPT(gpt_config)
|
||||||
|
|
||||||
# Forward pass
|
# Forward pass
|
||||||
logits = model(random_inputs)
|
logits, _ = model(random_inputs)
|
||||||
|
|
||||||
# Check output shape
|
# Check output shape
|
||||||
batch_size, seq_len = random_inputs.shape
|
batch_size, seq_len = random_inputs.shape
|
||||||
@@ -45,7 +45,7 @@ class TestGPT:
|
|||||||
model = GPT(gpt_config)
|
model = GPT(gpt_config)
|
||||||
|
|
||||||
# Forward pass with mask
|
# Forward pass with mask
|
||||||
logits = model(random_inputs, attention_mask=attention_mask)
|
logits, _ = model(random_inputs, attention_mask=attention_mask)
|
||||||
|
|
||||||
# Check output shape
|
# Check output shape
|
||||||
batch_size, seq_len = random_inputs.shape
|
batch_size, seq_len = random_inputs.shape
|
||||||
@@ -132,7 +132,7 @@ class TestGPT:
|
|||||||
model = GPT(gpt_config)
|
model = GPT(gpt_config)
|
||||||
|
|
||||||
# Forward pass
|
# Forward pass
|
||||||
logits = model(random_inputs)
|
logits, _ = model(random_inputs)
|
||||||
|
|
||||||
# Create a dummy loss and backward pass
|
# Create a dummy loss and backward pass
|
||||||
targets = torch.randint(0, gpt_config["vocab_size"], random_inputs.shape)
|
targets = torch.randint(0, gpt_config["vocab_size"], random_inputs.shape)
|
||||||
@@ -157,7 +157,7 @@ class TestGPT:
|
|||||||
inputs = random_inputs.to(device)
|
inputs = random_inputs.to(device)
|
||||||
|
|
||||||
# Forward pass
|
# Forward pass
|
||||||
logits = model(inputs)
|
logits, _ = model(inputs)
|
||||||
|
|
||||||
# Check device consistency
|
# Check device consistency
|
||||||
assert logits.device == device
|
assert logits.device == device
|
||||||
@@ -197,7 +197,7 @@ class TestGPT:
|
|||||||
batch_size, seq_len = 2, 16
|
batch_size, seq_len = 2, 16
|
||||||
inputs = torch.randint(0, config["vocab_size"], (batch_size, seq_len))
|
inputs = torch.randint(0, config["vocab_size"], (batch_size, seq_len))
|
||||||
|
|
||||||
logits = model(inputs)
|
logits, _ = model(inputs)
|
||||||
|
|
||||||
expected_shape = (batch_size, seq_len, config["vocab_size"])
|
expected_shape = (batch_size, seq_len, config["vocab_size"])
|
||||||
assert logits.shape == expected_shape
|
assert logits.shape == expected_shape
|
||||||
@@ -208,7 +208,7 @@ class TestGPT:
|
|||||||
model = GPT(gpt_config)
|
model = GPT(gpt_config)
|
||||||
|
|
||||||
inputs = torch.randint(0, gpt_config["vocab_size"], (batch_size, seq_len))
|
inputs = torch.randint(0, gpt_config["vocab_size"], (batch_size, seq_len))
|
||||||
logits = model(inputs)
|
logits, _ = model(inputs)
|
||||||
|
|
||||||
expected_shape = (batch_size, seq_len, gpt_config["vocab_size"])
|
expected_shape = (batch_size, seq_len, gpt_config["vocab_size"])
|
||||||
assert logits.shape == expected_shape
|
assert logits.shape == expected_shape
|
||||||
@@ -219,11 +219,11 @@ class TestGPT:
|
|||||||
|
|
||||||
# Training mode
|
# Training mode
|
||||||
model.train()
|
model.train()
|
||||||
output_train = model(random_inputs)
|
output_train, _ = model(random_inputs)
|
||||||
|
|
||||||
# Evaluation mode
|
# Evaluation mode
|
||||||
model.eval()
|
model.eval()
|
||||||
output_eval = model(random_inputs)
|
output_eval, _ = model(random_inputs)
|
||||||
|
|
||||||
# Outputs should be different due to dropout
|
# Outputs should be different due to dropout
|
||||||
assert not torch.allclose(output_train, output_eval)
|
assert not torch.allclose(output_train, output_eval)
|
||||||
@@ -271,7 +271,7 @@ class TestGPT:
|
|||||||
"""Test that GPT output has proper distribution."""
|
"""Test that GPT output has proper distribution."""
|
||||||
model = GPT(gpt_config)
|
model = GPT(gpt_config)
|
||||||
|
|
||||||
logits = model(random_inputs)
|
logits, _ = model(random_inputs)
|
||||||
|
|
||||||
# Logits should not have extreme values
|
# Logits should not have extreme values
|
||||||
assert logits.abs().max() < 100
|
assert logits.abs().max() < 100
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ def test_gpt_model_creation():
|
|||||||
input_ids = torch.randint(0, config["vocab_size"], (batch_size, seq_len))
|
input_ids = torch.randint(0, config["vocab_size"], (batch_size, seq_len))
|
||||||
|
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
logits = model(input_ids)
|
logits, _ = model(input_ids)
|
||||||
|
|
||||||
assert logits.shape == (batch_size, seq_len, config["vocab_size"])
|
assert logits.shape == (batch_size, seq_len, config["vocab_size"])
|
||||||
print("✅ GPT model creation and forward pass test passed")
|
print("✅ GPT model creation and forward pass test passed")
|
||||||
@@ -222,7 +222,7 @@ def test_gpt_with_tokenizer():
|
|||||||
input_ids = torch.tensor([tokens])
|
input_ids = torch.tensor([tokens])
|
||||||
|
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
logits = model(input_ids)
|
logits, _ = model(input_ids)
|
||||||
|
|
||||||
assert logits.shape == (1, len(tokens), vocab_size)
|
assert logits.shape == (1, len(tokens), vocab_size)
|
||||||
print("✅ GPT with tokenizer integration test passed")
|
print("✅ GPT with tokenizer integration test passed")
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user