bitsandbytes/pr_1457/en/_app/immutable/nodes/16.aad4f3ea.js

import{s as ge,o as _e,n as ue}from"../chunks/scheduler.852ec091.js";import{S as ye,i as fe,g as m,s,r as f,A as he,h as l,f as t,c as o,j as F,u as h,x as P,k as I,y as c,a as i,v,d as $,t as x,w}from"../chunks/index.28275fd3.js";import{D as K}from"../chunks/Docstring.ee6c313e.js";import{C as ve}from"../chunks/CodeBlock.c3366071.js";import{E as $e}from"../chunks/ExampleCodeBlock.00f06ed4.js";import{H as se,E as xe}from"../chunks/EditOnGithub.582011f0.js";function we(R){let d,M="Example:",u,b,p;return b=new ve({props:{code:"JTIzJTIwSW5pdGlhbGl6ZSUyMFN0YWJsZUVtYmVkZGluZyUyMGxheWVyJTIwd2l0aCUyMHZvY2FidWxhcnklMjBzaXplJTIwMTAwMCUyQyUyMGVtYmVkZGluZyUyMGRpbWVuc2lvbiUyMDMwMCUwQWVtYmVkZGluZ19sYXllciUyMCUzRCUyMFN0YWJsZUVtYmVkZGluZyhudW1fZW1iZWRkaW5ncyUzRDEwMDAlMkMlMjBlbWJlZGRpbmdfZGltJTNEMzAwKSUwQSUwQSUyMyUyMFJlc2V0JTIwZW1iZWRkaW5nJTIwcGFyYW1ldGVycyUwQWVtYmVkZGluZ19sYXllci5yZXNldF9wYXJhbWV0ZXJzKCklMEElMEElMjMlMjBQZXJmb3JtJTIwYSUyMGZvcndhcmQlMjBwYXNzJTIwd2l0aCUyMGlucHV0JTIwdGVuc29yJTBBaW5wdXRfdGVuc29yJTIwJTNEJTIwdG9yY2gudGVuc29yKCU1QjElMkMlMjAyJTJDJTIwMyU1RCklMEFvdXRwdXRfZW1iZWRkaW5nJTIwJTNEJTIwZW1iZWRkaW5nX2xheWVyKGlucHV0X3RlbnNvcik=",highlighted:`# Initialize StableEmbedding layer <span class="hljs-keyword">with</span> vocabulary size <span class="hljs-number">1000</span>, embedding dimension <span class="hljs-number">300</span>
embedding_layer = StableEmbedding(num_embeddings=<span class="hljs-number">1000</span>, embedding_dim=<span class="hljs-number">300</span>)

# <span class="hljs-keyword">Reset</span> embedding parameters
embedding_layer.reset_parameters()

# <span class="hljs-keyword">Perform</span> a forward pass <span class="hljs-keyword">with</span> <span class="hljs-keyword">input</span> tensor
input_tensor = torch.tensor([<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>])
output_embedding = embedding_layer(input_tensor)`,wrap:!1}}),{c(){d=m("p"),d.textContent=M,u=s(),f(b.$$.fragment)},l(a){d=l(a,"P",{"data-svelte-h":!0}),P(d)!=="svelte-11lpom8"&&(d.textContent=M),u=o(a),h(b.$$.fragment,a)},m(a,g){i(a,d,g),i(a,u,g),v(b,a,g),p=!0},p:ue,i(a){p||($(b.$$.fragment,a),p=!0)},o(a){x(b.$$.fragment,a),p=!1},d(a){a&&(t(d),t(u)),w(b,a)}}}function Ee(R){let d,M,u,b,p,a,g,oe="The embedding class is used to store and retrieve word embeddings from their indices. There are two types of embeddings in bitsandbytes, the standard PyTorch <code>Embedding</code> class and the <code>StableEmbedding</code> class.",X,S,re='The <code>StableEmbedding</code> class was introduced in the <a href="https://hf.co/papers/2110.02861" rel="nofollow">8-bit Optimizers via Block-wise Quantization</a> paper to reduce gradient variance as a result of the non-uniform distribution of input tokens. This class is designed to support quantization.',Y,k,L,_,C,ee,V,me="Embedding class to store and retrieve word embeddings from their indices.",ne,U,J,D,N,B,r,Z,te,G,le="Custom embedding layer designed to improve stability during training for NLP tasks by using 32-bit optimizer states. It is designed to reduce gradient variations that can result from quantization. This embedding layer is initialized with Xavier uniform initialization followed by layer normalization.",ae,T,ie,O,be=`Methods:
reset_parameters(): Reset embedding parameters using Xavier uniform initialization.
forward(input: Tensor) -> Tensor: Forward pass through the stable embedding layer.`,de,j,z,H,W,Q,q,A;return p=new se({props:{title:"Embedding",local:"embedding",headingTag:"h1"}}),k=new se({props:{title:"Embedding",local:"bitsandbytes.nn.Embedding",headingTag:"h2"}}),C=new K({props:{name:"class bitsandbytes.nn.Embedding",anchor:"bitsandbytes.nn.Embedding",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:": typing.Optional[torch.device] = None"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1457/bitsandbytes/nn/modules.py#L131"}}),J=new K({props:{name:"__init__",anchor:"bitsandbytes.nn.Embedding.__init__",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:": typing.Optional[torch.device] = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.Embedding.__init__.num_embeddings",description:`<strong>num_embeddings</strong> (<code>int</code>) &#x2014;
The number of unique embeddings (vocabulary size).`,name:"num_embeddings"},{anchor:"bitsandbytes.nn.Embedding.__init__.embedding_dim",description:`<strong>embedding_dim</strong> (<code>int</code>) &#x2014;
The dimensionality of the embedding.`,name:"embedding_dim"},{anchor:"bitsandbytes.nn.Embedding.__init__.padding_idx",description:`<strong>padding_idx</strong> (<code>Optional[int]</code>) &#x2014;
Pads the output with zeros at the given index.`,name:"padding_idx"},{anchor:"bitsandbytes.nn.Embedding.__init__.max_norm",description:`<strong>max_norm</strong> (<code>Optional[float]</code>) &#x2014;
Renormalizes embeddings to have a maximum L2 norm.`,name:"max_norm"},{anchor:"bitsandbytes.nn.Embedding.__init__.norm_type",description:`<strong>norm_type</strong> (<code>float</code>, defaults to <code>2.0</code>) &#x2014;
The p-norm to compute for the <code>max_norm</code> option.`,name:"norm_type"},{anchor:"bitsandbytes.nn.Embedding.__init__.scale_grad_by_freq",description:`<strong>scale_grad_by_freq</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Scale gradient by frequency during backpropagation.`,name:"scale_grad_by_freq"},{anchor:"bitsandbytes.nn.Embedding.__init__.sparse",description:`<strong>sparse</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Computes dense gradients. Set to <code>True</code> to compute sparse gradients instead.`,name:"sparse"},{anchor:"bitsandbytes.nn.Embedding.__init__._weight",description:`<strong>_weight</strong> (<code>Optional[Tensor]</code>) &#x2014;
Pretrained embeddings.`,name:"_weight"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1457/bitsandbytes/nn/modules.py#L136"}}),N=new se({props:{title:"StableEmbedding",local:"bitsandbytes.nn.StableEmbedding",headingTag:"h2"}}),Z=new K({props:{name:"class bitsandbytes.nn.StableEmbedding",anchor:"bitsandbytes.nn.StableEmbedding",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:" = None"},{name:"dtype",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.StableEmbedding.norm",description:"<strong>norm</strong> (<code>torch.nn.LayerNorm</code>) &#x2014; Layer normalization applied after the embedding.",name:"norm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1457/bitsandbytes/nn/modules.py#L25"}}),T=new $e({props:{anchor:"bitsandbytes.nn.StableEmbedding.example",$$slots:{default:[we]},$$scope:{ctx:R}}}),z=new K({props:{name:"__init__",anchor:"bitsandbytes.nn.StableEmbedding.__init__",parameters:[{name:"num_embeddings",val:": int"},{name:"embedding_dim",val:": int"},{name:"padding_idx",val:": typing.Optional[int] = None"},{name:"max_norm",val:": typing.Optional[float] = None"},{name:"norm_type",val:": float = 2.0"},{name:"scale_grad_by_freq",val:": bool = False"},{name:"sparse",val:": bool = False"},{name:"_weight",val:": typing.Optional[torch.Tensor] = None"},{name:"device",val:" = None"},{name:"dtype",val:" = None"}],parametersDescription:[{anchor:"bitsandbytes.nn.StableEmbedding.__init__.num_embeddings",description:`<strong>num_embeddings</strong> (<code>int</code>) &#x2014;
The number of unique embeddings (vocabulary size).`,name:"num_embeddings"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.embedding_dim",description:`<strong>embedding_dim</strong> (<code>int</code>) &#x2014;
The dimensionality of the embedding.`,name:"embedding_dim"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.padding_idx",description:`<strong>padding_idx</strong> (<code>Optional[int]</code>) &#x2014;
Pads the output with zeros at the given index.`,name:"padding_idx"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.max_norm",description:`<strong>max_norm</strong> (<code>Optional[float]</code>) &#x2014;
Renormalizes embeddings to have a maximum L2 norm.`,name:"max_norm"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.norm_type",description:`<strong>norm_type</strong> (<code>float</code>, defaults to <code>2.0</code>) &#x2014;
The p-norm to compute for the <code>max_norm</code> option.`,name:"norm_type"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.scale_grad_by_freq",description:`<strong>scale_grad_by_freq</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Scale gradient by frequency during backpropagation.`,name:"scale_grad_by_freq"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__.sparse",description:`<strong>sparse</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Computes dense gradients. Set to <code>True</code> to compute sparse gradients instead.`,name:"sparse"},{anchor:"bitsandbytes.nn.StableEmbedding.__init__._weight",description:`<strong>_weight</strong> (<code>Optional[Tensor]</code>) &#x2014;
Pretrained embeddings.`,name:"_weight"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1457/bitsandbytes/nn/modules.py#L51"}}),W=new xe({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/nn/embeddings.mdx"}}),{c(){d=m("meta"),M=s(),u=m("p"),b=s(),f(p.$$.fragment),a=s(),g=m("p"),g.innerHTML=oe,X=s(),S=m("p"),S.innerHTML=re,Y=s(),f(k.$$.fragment),L=s(),_=m("div"),f(C.$$.fragment),ee=s(),V=m("p"),V.textContent=me,ne=s(),U=m("div"),f(J.$$.fragment),D=s(),f(N.$$.fragment),B=s(),r=m("div"),f(Z.$$.fragment),te=s(),G=m("p"),G.textContent=le,ae=s(),f(T.$$.fragment),ie=s(),O=m("p"),O.textContent=be,de=s(),j=m("div"),f(z.$$.fragment),H=s(),f(W.$$.fragment),Q=s(),q=m("p"),this.h()},l(e){const n=he("svelte-u9bgzb",document.head);d=l(n,"META",{name:!0,content:!0}),n.forEach(t),M=o(e),u=l(e,"P",{}),F(u).forEach(t),b=o(e),h(p.$$.fragment,e),a=o(e),g=l(e,"P",{"data-svelte-h":!0}),P(g)!=="svelte-hjtctc"&&(g.innerHTML=oe),X=o(e),S=l(e,"P",{"data-svelte-h":!0}),P(S)!=="svelte-1vvt62i"&&(S.innerHTML=re),Y=o(e),h(k.$$.fragment,e),L=o(e),_=l(e,"DIV",{class:!0});var E=F(_);h(C.$$.fragment,E),ee=o(E),V=l(E,"P",{"data-svelte-h":!0}),P(V)!=="svelte-1tikli9"&&(V.textContent=me),ne=o(E),U=l(E,"DIV",{class:!0});var pe=F(U);h(J.$$.fragment,pe),pe.forEach(t),E.forEach(t),D=o(e),h(N.$$.fragment,e),B=o(e),r=l(e,"DIV",{class:!0});var y=F(r);h(Z.$$.fragment,y),te=o(y),G=l(y,"P",{"data-svelte-h":!0}),P(G)!=="svelte-15pctl3"&&(G.textContent=le),ae=o(y),h(T.$$.fragment,y),ie=o(y),O=l(y,"P",{"data-svelte-h":!0}),P(O)!=="svelte-wb92uq"&&(O.textContent=be),de=o(y),j=l(y,"DIV",{class:!0});var ce=F(j);h(z.$$.fragment,ce),ce.forEach(t),y.forEach(t),H=o(e),h(W.$$.fragment,e),Q=o(e),q=l(e,"P",{}),F(q).forEach(t),this.h()},h(){I(d,"name","hf:doc:metadata"),I(d,"content",Te),I(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),I(_,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),I(j,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),I(r,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,n){c(document.head,d),i(e,M,n),i(e,u,n),i(e,b,n),v(p,e,n),i(e,a,n),i(e,g,n),i(e,X,n),i(e,S,n),i(e,Y,n),v(k,e,n),i(e,L,n),i(e,_,n),v(C,_,null),c(_,ee),c(_,V),c(_,ne),c(_,U),v(J,U,null),i(e,D,n),v(N,e,n),i(e,B,n),i(e,r,n),v(Z,r,null),c(r,te),c(r,G),c(r,ae),v(T,r,null),c(r,ie),c(r,O),c(r,de),c(r,j),v(z,j,null),i(e,H,n),v(W,e,n),i(e,Q,n),i(e,q,n),A=!0},p(e,[n]){const E={};n&2&&(E.$$scope={dirty:n,ctx:e}),T.$set(E)},i(e){A||($(p.$$.fragment,e),$(k.$$.fragment,e),$(C.$$.fragment,e),$(J.$$.fragment,e),$(N.$$.fragment,e),$(Z.$$.fragment,e),$(T.$$.fragment,e),$(z.$$.fragment,e),$(W.$$.fragment,e),A=!0)},o(e){x(p.$$.fragment,e),x(k.$$.fragment,e),x(C.$$.fragment,e),x(J.$$.fragment,e),x(N.$$.fragment,e),x(Z.$$.fragment,e),x(T.$$.fragment,e),x(z.$$.fragment,e),x(W.$$.fragment,e),A=!1},d(e){e&&(t(M),t(u),t(b),t(a),t(g),t(X),t(S),t(Y),t(L),t(_),t(D),t(B),t(r),t(H),t(Q),t(q)),t(d),w(p,e),w(k,e),w(C),w(J),w(N,e),w(Z),w(T),w(z),w(W,e)}}}const Te='{"title":"Embedding","local":"embedding","sections":[{"title":"Embedding","local":"bitsandbytes.nn.Embedding","sections":[],"depth":2},{"title":"StableEmbedding","local":"bitsandbytes.nn.StableEmbedding","sections":[],"depth":2}],"depth":1}';function Me(R){return _e(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ze extends ye{constructor(d){super(),fe(this,d,Me,Ee,ge,{})}}export{ze as component};