bitsandbytes/pr_1187/en/_app/immutable/nodes/21.0e70ddea.js

import{s as vt,n as yt,o as $t}from"../chunks/scheduler.852ec091.js";import{S as xt,i as Lt,g as s,s as a,r as d,A as At,h as r,f as e,c as i,j as x,u as l,x as st,k as L,y as m,a as o,v as p,d as c,t as b,w as _}from"../chunks/index.28275fd3.js";import{D as V}from"../chunks/Docstring.395987e7.js";import{H as rt,E as St}from"../chunks/EditOnGithub.582011f0.js";function Rt(_t){let g,W,H,q,A,B,S,gt='<a href="https:/hf.co/papers/1708.03888" rel="nofollow">LARS (Layer-wise Adaptive Rate Scaling)</a> is an optimizer designed for training with large batch sizes to accelerate training. LARS uses a separate learning rate for each <em>layer</em> instead of each parameter. The learning rate is calculated from a <em>trust ratio</em> between the weight and gradient norm in a layer. This helps calibrate a stable update size.',G,R,O,u,w,mt,v,z,dt,F,ut="Base LARS optimizer.",U,T,J,h,E,lt,y,N,pt,j,ht="8-bit LARS optimizer.",K,C,Q,f,D,ct,$,P,bt,I,ft="32-bit LARS optimizer.",X,k,Y,M,Z;return A=new rt({props:{title:"LARS",local:"lars",headingTag:"h1"}}),R=new rt({props:{title:"LARS",local:"api-class ][ bitsandbytes.optim.LARS",headingTag:"h2"}}),w=new V({props:{name:"class bitsandbytes.optim.LARS",anchor:"bitsandbytes.optim.LARS",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"max_unorm",val:" = 0.02"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1187/bitsandbytes/optim/lars.py#L11"}}),z=new V({props:{name:"__init__",anchor:"bitsandbytes.optim.LARS.__init__",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"optim_bits",val:" = 32"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS.__init__.optim_bits",description:`<strong>optim_bits</strong> (<code>int</code>, defaults to 32) &#x2014;
The number of bits of the optimizer state.`,name:"optim_bits"},{anchor:"bitsandbytes.optim.LARS.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.LARS.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1187/bitsandbytes/optim/lars.py#L12"}}),T=new rt({props:{title:"LARS8bit",local:"bitsandbytes.optim.LARS8bit",headingTag:"h2"}}),E=new V({props:{name:"class bitsandbytes.optim.LARS8bit",anchor:"bitsandbytes.optim.LARS8bit",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"max_unorm",val:" = 0.02"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1187/bitsandbytes/optim/lars.py#L71"}}),N=new V({props:{name:"__init__",anchor:"bitsandbytes.optim.LARS8bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS8bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.LARS8bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1187/bitsandbytes/optim/lars.py#L72"}}),C=new rt({props:{title:"LARS32bit",local:"bitsandbytes.optim.LARS32bit",headingTag:"h2"}}),D=new V({props:{name:"class bitsandbytes.optim.LARS32bit",anchor:"bitsandbytes.optim.LARS32bit",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"max_unorm",val:" = 0.02"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1187/bitsandbytes/optim/lars.py#L128"}}),P=new V({props:{name:"__init__",anchor:"bitsandbytes.optim.LARS32bit.__init__",parameters:[{name:"params",val:""},{name:"lr",val:""},{name:"momentum",val:" = 0"},{name:"dampening",val:" = 0"},{name:"weight_decay",val:" = 0"},{name:"nesterov",val:" = False"},{name:"args",val:" = None"},{name:"min_8bit_size",val:" = 4096"},{name:"percentile_clipping",val:" = 100"},{name:"max_unorm",val:" = 0.02"}],parametersDescription:[{anchor:"bitsandbytes.optim.LARS32bit.__init__.params",description:`<strong>params</strong> (<code>torch.tensor</code>) &#x2014;
The input parameters to optimize.`,name:"params"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.lr",description:`<strong>lr</strong> (<code>float</code>) &#x2014;
The learning rate.`,name:"lr"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.momentum",description:`<strong>momentum</strong> (<code>float</code>, defaults to 0) &#x2014;
The momentum value speeds up the optimizer by taking bigger steps.`,name:"momentum"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.dampening",description:`<strong>dampening</strong> (<code>float</code>, defaults to 0) &#x2014;
The dampening value reduces the momentum of the optimizer.`,name:"dampening"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.weight_decay",description:`<strong>weight_decay</strong> (<code>float</code>, defaults to 1e-2) &#x2014;
The weight decay value for the optimizer.`,name:"weight_decay"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.nesterov",description:`<strong>nesterov</strong> (<code>bool</code>, defaults to <code>False</code>) &#x2014;
Whether to use Nesterov momentum.`,name:"nesterov"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.args",description:`<strong>args</strong> (<code>object</code>, defaults to <code>None</code>) &#x2014;
An object with additional arguments.`,name:"args"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.min_8bit_size",description:`<strong>min_8bit_size</strong> (<code>int</code>, defaults to 4096) &#x2014;
The minimum number of elements of the parameter tensors for 8-bit optimization.`,name:"min_8bit_size"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.percentile_clipping",description:`<strong>percentile_clipping</strong> (<code>int</code>, defaults to 100) &#x2014;
Adapts clipping threshold automatically by tracking the last 100 gradient norms and clipping the gradient at a certain percentile to improve stability.`,name:"percentile_clipping"},{anchor:"bitsandbytes.optim.LARS32bit.__init__.max_unorm",description:`<strong>max_unorm</strong> (<code>float</code>, defaults to 0.02) &#x2014;
The maximum gradient norm.`,name:"max_unorm"}],source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/vr_1187/bitsandbytes/optim/lars.py#L129"}}),k=new St({props:{source:"https://github.com/bitsandbytes-foundation/bitsandbytes/blob/main/docs/source/reference/optim/lars.mdx"}}),{c(){g=s("meta"),W=a(),H=s("p"),q=a(),d(A.$$.fragment),B=a(),S=s("p"),S.innerHTML=gt,G=a(),d(R.$$.fragment),O=a(),u=s("div"),d(w.$$.fragment),mt=a(),v=s("div"),d(z.$$.fragment),dt=a(),F=s("p"),F.textContent=ut,U=a(),d(T.$$.fragment),J=a(),h=s("div"),d(E.$$.fragment),lt=a(),y=s("div"),d(N.$$.fragment),pt=a(),j=s("p"),j.textContent=ht,K=a(),d(C.$$.fragment),Q=a(),f=s("div"),d(D.$$.fragment),ct=a(),$=s("div"),d(P.$$.fragment),bt=a(),I=s("p"),I.textContent=ft,X=a(),d(k.$$.fragment),Y=a(),M=s("p"),this.h()},l(t){const n=At("svelte-u9bgzb",document.head);g=r(n,"META",{name:!0,content:!0}),n.forEach(e),W=i(t),H=r(t,"P",{}),x(H).forEach(e),q=i(t),l(A.$$.fragment,t),B=i(t),S=r(t,"P",{"data-svelte-h":!0}),st(S)!=="svelte-18efkb3"&&(S.innerHTML=gt),G=i(t),l(R.$$.fragment,t),O=i(t),u=r(t,"DIV",{class:!0});var tt=x(u);l(w.$$.fragment,tt),mt=i(tt),v=r(tt,"DIV",{class:!0});var et=x(v);l(z.$$.fragment,et),dt=i(et),F=r(et,"P",{"data-svelte-h":!0}),st(F)!=="svelte-1vko6mi"&&(F.textContent=ut),et.forEach(e),tt.forEach(e),U=i(t),l(T.$$.fragment,t),J=i(t),h=r(t,"DIV",{class:!0});var nt=x(h);l(E.$$.fragment,nt),lt=i(nt),y=r(nt,"DIV",{class:!0});var at=x(y);l(N.$$.fragment,at),pt=i(at),j=r(at,"P",{"data-svelte-h":!0}),st(j)!=="svelte-mgnzq1"&&(j.textContent=ht),at.forEach(e),nt.forEach(e),K=i(t),l(C.$$.fragment,t),Q=i(t),f=r(t,"DIV",{class:!0});var it=x(f);l(D.$$.fragment,it),ct=i(it),$=r(it,"DIV",{class:!0});var ot=x($);l(P.$$.fragment,ot),bt=i(ot),I=r(ot,"P",{"data-svelte-h":!0}),st(I)!=="svelte-1l89br0"&&(I.textContent=ft),ot.forEach(e),it.forEach(e),X=i(t),l(k.$$.fragment,t),Y=i(t),M=r(t,"P",{}),x(M).forEach(e),this.h()},h(){L(g,"name","hf:doc:metadata"),L(g,"content",wt),L(v,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(u,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(h,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),L(f,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(t,n){m(document.head,g),o(t,W,n),o(t,H,n),o(t,q,n),p(A,t,n),o(t,B,n),o(t,S,n),o(t,G,n),p(R,t,n),o(t,O,n),o(t,u,n),p(w,u,null),m(u,mt),m(u,v),p(z,v,null),m(v,dt),m(v,F),o(t,U,n),p(T,t,n),o(t,J,n),o(t,h,n),p(E,h,null),m(h,lt),m(h,y),p(N,y,null),m(y,pt),m(y,j),o(t,K,n),p(C,t,n),o(t,Q,n),o(t,f,n),p(D,f,null),m(f,ct),m(f,$),p(P,$,null),m($,bt),m($,I),o(t,X,n),p(k,t,n),o(t,Y,n),o(t,M,n),Z=!0},p:yt,i(t){Z||(c(A.$$.fragment,t),c(R.$$.fragment,t),c(w.$$.fragment,t),c(z.$$.fragment,t),c(T.$$.fragment,t),c(E.$$.fragment,t),c(N.$$.fragment,t),c(C.$$.fragment,t),c(D.$$.fragment,t),c(P.$$.fragment,t),c(k.$$.fragment,t),Z=!0)},o(t){b(A.$$.fragment,t),b(R.$$.fragment,t),b(w.$$.fragment,t),b(z.$$.fragment,t),b(T.$$.fragment,t),b(E.$$.fragment,t),b(N.$$.fragment,t),b(C.$$.fragment,t),b(D.$$.fragment,t),b(P.$$.fragment,t),b(k.$$.fragment,t),Z=!1},d(t){t&&(e(W),e(H),e(q),e(B),e(S),e(G),e(O),e(u),e(U),e(J),e(h),e(K),e(Q),e(f),e(X),e(Y),e(M)),e(g),_(A,t),_(R,t),_(w),_(z),_(T,t),_(E),_(N),_(C,t),_(D),_(P),_(k,t)}}}const wt='{"title":"LARS","local":"lars","sections":[{"title":"LARS","local":"api-class ][ bitsandbytes.optim.LARS","sections":[],"depth":2},{"title":"LARS8bit","local":"bitsandbytes.optim.LARS8bit","sections":[],"depth":2},{"title":"LARS32bit","local":"bitsandbytes.optim.LARS32bit","sections":[],"depth":2}],"depth":1}';function zt(_t){return $t(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Dt extends xt{constructor(g){super(),Lt(this,g,zt,Rt,vt,{})}}export{Dt as component};