ezt ird at polinomokra: \[ \begin{gathered} \operatorname{fl...

Prompt

ezt ird at polinomokra: \[ \begin{gathered} \operatorname{fl}_{32}(x)=\text{binary32-round}(x),\qquad S(x)= \begin{cases} 0,&x\in\{\mathrm{NaN},+\infty,-\infty\},\\ x,&\text{otherwise}, \end{cases} \qquad C(x)=\max_{32}\!\left(-60000,\min_{32}(60000,S(x))\right),\\ c=0.70710677_{\!32},\qquad \operatorname{f16}(x)=\text{binary16-round}(\operatorname{fl}_{32}(x)). \end{gathered} \] \[ \begin{aligned} \operatorname{clip\_matrix}(X,\rho):\quad& m=\operatorname{ReduceMax}_{32}\bigl(\{|X_{ij}|\}\cup\{0\}\bigr),\\ &h= \begin{cases} \operatorname{ReduceAdd}_{32}\!\left(\left\{\operatorname{fl}_{32}((X_{ij}/m)^2)\right\}\right),&m>0,\\ 0,&m\le 0, \end{cases}\\ &n=\operatorname{fl}_{32}\!\left(m\sqrt{h}\right),\qquad s= \begin{cases} \operatorname{fl}_{32}(\rho/n),&0<\rho<n\ \land\ 10^{-12}<n,\\ 1,&\text{otherwise}, \end{cases} \qquad Y_{ij}=\operatorname{fl}_{32}(sX_{ij}). \end{aligned} \] \[ \begin{aligned} \operatorname{embedding\_backward}(T,\ell,G,W)_{v,d} &=W_{v,d}+ \sum_{\substack{b,s:\\0\le s<\ell_b\\ T_{\sum_{q<b}\ell_q+s}=v}}G_{b,s,d},\\[2mm] L_b&=\max(0,\min(S,\ell_b)),\qquad q_s= \begin{cases}r_s,&0\le r_s<S,\\s,&\text{otherwise},\end{cases}\\ j_{b,s}&=Sb+q_s,\qquad \operatorname{embedding\_forward}(T,\ell,r,W)_{b,s,d} = \begin{cases} W_{T_{j_{b,s}},d},& 0\le q_s<L_b\land 0\le j_{b,s}<|T| \land 0\le T_{j_{b,s}}<V,\\ 0_{\mathrm{f16}},&\text{otherwise}. \end{cases} \end{aligned} \] \[ \begin{aligned} \operatorname{embedding\_spectral}(A,a,b,p,t):\quad& \epsilon=10^{-12},\quad K=\max(1,p),\\ &u_0= \begin{cases}a/\|a\|_2,&\|a\|_2>\epsilon,\\ \mathbf1/\sqrt V,&\text{otherwise},\end{cases} \qquad v_0= \begin{cases}b/\|b\|_2,&\|b\|_2>\epsilon,\\ \mathbf1/\sqrt D,&\text{otherwise},\end{cases}\\ &q_k=A^\top u_k,\qquad v'_k= \begin{cases}q_k/\|q_k\|_2,&\|q_k\|_2>\epsilon,\\v_k,&\text{otherwise},\end{cases}\\ &u_{k+1}=Av'_k/\|Av'_k\|_2,\qquad v_{k+1}=A^\top u_{k+1}/\|A^\top u_{k+1}\|_2,\qquad d=u_K^\top Av_K,\\ &\sigma=S(|d|),\quad \alpha= \begin{cases} \max_{32}(10^{-6},t)/\sigma,&\max_{32}(10^{-6},t)<\sigma,\\ 1,&\text{otherwise}, \end{cases}\\ &(A',u',v',s_1,s_2)=(\alpha A,u_K,v_K,\sigma,\sigma\alpha). \end{aligned} \] \[ \begin{aligned} \operatorname{embedding\_sum\_squares}(A) &=S\!\left( \operatorname{ReduceAdd}_{32} \left\{ \operatorname{fl}_{32}\!\left(S(\operatorname{f32}(A_{ij}))^2\right) \right\}_{i,j} \right),\\ \beta&=\max_{32}(0,\min_{32}(0.99999994,S(\beta_0))),\quad \gamma=\max_{32}(0,\min_{32}(0.99999994,S(\gamma_0))),\\ \epsilon&=\max_{32}(10^{-12},S(\epsilon_0)),\quad \rho=\max_{32}(0,\min_{32}(1,S(\rho_0))),\quad q=\max_{32}(0,S(q_0)),\\ m'&=S\!\left(\beta m+(1-\beta)g\right),\\ f_0&=\max_{32}(0,\min_{32}(10^6,S(f))),\qquad f'=\max_{32}\!\left(0,\min_{32}\!\left(10^6,S\!\left(\gamma f_0+g((1-\gamma)g)\right)\right)\right),\\ t&=\max(1,\text{step}),\quad \hat m=\frac{m'}{\max_{32}(\epsilon,1-\beta^t)},\quad \hat f=\frac{f'}{\max_{32}(\epsilon,1-\gamma^t)},\\ \delta&=\max_{32}\!\left(-\rho\max_{32}(q,|w|), \min_{32}\!\left(\rho\max_{32}(q,|w|), S\!\left(\frac{S(\mathrm{lr})\hat m}{\epsilon+\sqrt{\max_{32}(0,\hat f)}}\right)\right)\right),\\ (w',m',f')&=(S(w-\delta),m',f'). \end{aligned} \] \[ \begin{aligned} z_0&=(v+0x9e3779b97f4a7c15)\bmod2^{64},\\ z_1&=((z_0\mathbin{\mathrm{xor}}(z_0\!\gg\!30))\,0xbf58476d1ce4e5b9)\bmod2^{64},\\ z_2&=((z_1\mathbin{\mathrm{xor}}(z_1\!\gg\!27))\,0x94d049bb133111eb)\bmod2^{64},\\ M(v)&=z_2\mathbin{\mathrm{xor}}(z_2\!\gg\!31),\quad q=M(\mathrm{seed}),\quad x_i=M(q\mathbin{\mathrm{xor}}h_i),\quad y_i=M(x_i),\\ \theta(r)&=\operatorname{fl}_{32}\!\left(10^{-6}\operatorname{fl}_{32} \!\left(6.2831855\,(r\bmod10^6)\right)\right),\\ (c_0,s_0,c_1,s_1)&= \left(\cos\theta(x_i),\sin\theta(x_i\!\gg\!20), \cos\theta(x_i\!\gg\!40),\sin\theta(y_i)\right),\\ d_i&=\sqrt{\sum_{\xi\in\{c_0,s_0,c_1,s_1\}}\xi^2}\ \text{ if }\sum\xi^2>10^{-30}, \quad d_i=1\ \text{otherwise},\\ (\mathrm{out}_0[i],\mathrm{out}_1[i],\ldots,\mathrm{out}_4[i]) &=\left(h_i,\frac{c_0}{d_i},\frac{s_0}{d_i},\frac{c_1}{d_i},\frac{s_1}{d_i}\right),\\ (\mathrm{out}_5[j],\mathrm{out}_6[j]) &= \begin{cases} (i,\ i-k-1),&i=\lfloor j/3\rfloor,\ k=j\bmod3,\ k<i,\\ (-1,-1),&\text{otherwise}. \end{cases} \end{aligned} \] \[ \begin{aligned} \operatorname{master\_weights\_to\_f16\_2d}(X)_{r,c} &=\operatorname{f16}\!\left(\max_{32}(-65504,\min_{32}(65504,S(X_{r,c})))\right),\\ \operatorname{master\_weights\_to\_f16\_3d}(X)_{\ell,r,c} &=\operatorname{f16}\!\left(\max_{32}(-65504,\min_{32}(65504,S(X_{\ell,r,c})))\right),\\ \operatorname{matmul}(A,B)_{ij} &=\operatorname{fold}_{r=0}^{k-1} \left(a\leftarrow\operatorname{fl}_{32}\!\left(a+ \operatorname{fl}_{32}(A_{ir}B_{rj})\right)\right),\qquad a_0=0,\\ \operatorname{scale\_matrix}(X,\lambda)_{ij} &=\operatorname{fl}_{32}(\lambda X_{ij}). \end{aligned} \] \[ \begin{aligned} \mathcal{D}(z;B,R,S,\mathrm{diff}) &=\begin{cases} z,&\mathrm{diff}=0\ \lor\ B\le0,\\ z^{(S)},&\mathrm{diff}=1,\ B>0, \end{cases}\\ z^{(s+1)}_i &= \begin{cases} \operatorname{fl}_{32}\!\left(c\,\operatorname{fl}_{32}(z^{(s)}_i+z^{(s)}_{i+2^s})\right), &\left\lfloor((i\bmod B)/2^s)\right\rfloor\bmod2=0,\\ \operatorname{fl}_{32}\!\left(c\,\operatorname{fl}_{32}(z^{(s)}_{i-2^s}-z^{(s)}_i)\right), &\text{otherwise}, \end{cases}\\ \mathcal{R}(z;B,R)_i &= \begin{cases} \operatorname{fl}_{32}\!\left(z_i-\operatorname{fl}_{32}\!\left( \operatorname{fl}_{32}(2/R)\displaystyle\sum_{r=0}^{R-1}z_{(i\bmod B)+Br}\right)\right), &B>0,\ R>0,\\ z_i,&\text{otherwise}. \end{cases} \end{aligned} \] \[ \begin{aligned} A_{b,t,i}&=\operatorname{fold}_{j=0}^{T-1} \left(s\leftarrow\operatorname{fl}_{32}\!\left(s+ \mathbf1_{M_{tj}\ne0}\,X_{b,j,H+i}\right)\right),\quad s_0=0,\\ q_{b,t,i}&=\operatorname{fl}_{32}\!\left(W^s_{i0}A_{b,t,i}+W^s_{i1}\right),\qquad a_{b,t,i}=X_{b,t,i}\exp\!\left(\max(\ell,\min(u,q_{b,t,i}))\right),\\ e_{b,t,i}&=S\!\left(\operatorname{fl}_{32}(W^t_{i0}a_{b,t,i}+W^t_{i1})\right),\qquad v_{b,t,i}=X_{b,t,H+i}+e_{b,t,i},\\ z_{b,t,i}&=\operatorname{fl}_{32}\!\left(c(a_{b,t,i}-v_{b,t,i})\right),\qquad z_{b,t,H+i}=\operatorname{fl}_{32}\!\left(c(a_{b,t,i}+v_{b,t,i})\right),\\ \operatorname{causal\_forward}(X,M,W^s,W^t) &=\operatorname{f16}\!\left(C\!\left(\mathcal R(\mathcal D(z;B,R,S,\mathrm{diff});B,R)\right)\right),\\ \operatorname{causal\_forward\_stack}(X,M,W^s,W^t) &=\operatorname{f16}(X^{(L)}),\quad X^{(0)}=S(\operatorname{f32}(X)),\\ X^{(\ell+1)} &=C\!\left(\mathcal R\!\left(\mathcal D\!\left( \operatorname{causal\_coupling}(X^{(\ell)},M,W^s_\ell,W^t_\ell); B,R,S,\mathrm{diff}\right);B,R\right)\right). \end{aligned} \] \[ \begin{aligned} z'&=\mathcal R(\mathcal D(S(\operatorname{f32}(Y));B,R,S,\mathrm{diff});B,R),\\ a_i&=c(z'_i+z'_{H+i}),\qquad d_i=c(z'_{H+i}-z'_i),\\ t_i&=S(W^t_{i0}a_i+W^t_{i1}),\qquad u_i=d_i-t_i,\\ r_{b,t,i}&=\operatorname{fold}_{j=0}^{T-1} \left(r\leftarrow\operatorname{fl}_{32}\!\left(r+ \mathbf1_{M_{jt}\ne0}\,u_{b,j,i}\right)\right),\quad r_0=0,\\ v_i&=\frac{a_i}{\exp(\max(\ell,\min(u_{\max},W^s_{i0}r_{b,t,i}+W^s_{i1})))} ,\\ \operatorname{causal\_inverse}(Y,M,W^s,W^t)_{b,t,:} &=\operatorname{f16}\!\left(C([v_0,\ldots,v_{H-1},u_0,\ldots,u_{H-1}])\right),\\ \operatorname{causal\_inverse\_stack}(Y,M,W^s,W^t) &=\operatorname{f16}(X^{(0)}),\qquad X^{(L)}=S(\operatorname{f32}(Y)),\\ X^{(\ell)}&=\operatorname{inverse\_coupling}_{\ell} \!\left(\mathcal R(\mathcal D(X^{(\ell+1)};B,R,S,\mathrm{diff});B,R),M\right), \quad \ell=L-1,\ldots,0. \end{aligned} \] \[ \begin{aligned} a_i&=\max(\ell,\min(u,W^s_{i0}X_{n,H+i}+W^s_{i1})),\\ q_i&=X_{n,i}\exp(a_i),\qquad t_i=S(W^t_{i0}q_i+W^t_{i1}),\qquad v_i=X_{n,H+i}+t_i,\\ z_{n,i}&=c(q_i-v_i),\qquad z_{n,H+i}=c(q_i+v_i),\\ \operatorname{rsf\_forward}(X,W^s,W^t) &=\operatorname{f16}\!\left(C\!\left(\mathcal R(\mathcal D(z;B,R,S,\mathrm{diff});B,R)\right)\right),\\ \operatorname{rsf\_stack\_forward}(X,W^s,W^t) &=\operatorname{f16}(X^{(L)}),\quad X^{(0)}=\operatorname{f32}(X),\\ X^{(\ell+1)} &=C\!\left(\mathcal R\!\left(\mathcal D\!\left( \operatorname{rsf\_coupling}(X^{(\ell)},W^s_\ell,W^t_\ell); B,R,S,\mathrm{diff}\right);B,R\right)\right),\\ \operatorname{rsf\_stack\_inverse}(Y,W^s,W^t) &=\operatorname{f16}(X^{(0)}),\quad X^{(L)}=\operatorname{f32}(Y),\\ X^{(\ell)} &=\operatorname{inverse\_coupling}_{\ell} \!\left(\mathcal R(\mathcal D(X^{(\ell+1)};B,R,S,\mathrm{diff});B,R)\right), \quad \ell=L-1,\ldots,0. \end{aligned} \] \[ \begin{aligned} \operatorname{rsf\_stack\_backward\_gradients\_fused}(\cdots) &=\mathcal B(\cdots),\\ \operatorname{rsf\_stack\_midpoint\_fused}(\cdots) &=\mathcal M(\cdots),\\ \operatorname{stack\_spectral\_normalize\_exact}(\cdots) &=\mathcal S(\cdots). \end{aligned} \] \[ \begin{aligned} \beta_1&=\max_{32}(0,\min_{32}(0.99999994,S(\beta_{1,0}))),\quad \beta_2=\max_{32}(0,\min_{32}(0.99999994,S(\beta_{2,0}))),\\ \epsilon&=\max_{32}(10^{-12},S(\epsilon_0)),\quad \eta=S(\mathrm{lr}),\quad \tau=\max_{32}(0,\min_{32}(1,S(\mathrm{trust}))),\quad f_0=\max_{32}(0,S(\mathrm{weight\_floor})),\\ m'_w&=S(\beta_1m_w+(1-\beta_1)g_w),\qquad m'_b=S(\beta_1m_b+(1-\beta_1)g_b),\\ F'_{ww}&=\operatorname{cap}_{d}(\beta_2F_{ww}+(1-\beta_2)g_w^2),\\ F'_{wb}&=\operatorname{cap}_{o}(\beta_2F_{wb}+(1-\beta_2)g_wg_b),\\ F'_{bb}&=\operatorname{cap}_{d}(\beta_2F_{bb}+(1-\beta_2)g_b^2),\\ \hat m_w&=\frac{m'_w}{\max(\epsilon,1-\beta_1^{\max(1,\mathrm{step})})},\quad \hat m_b=\frac{m'_b}{\max(\epsilon,1-\beta_1^{\max(1,\mathrm{step})})},\\ \hat F_{ww}&=\frac{F'_{ww}}{\max(\epsilon,1-\beta_2^{\max(1,\mathrm{step})})},\quad \hat F_{wb}=\frac{F'_{wb}}{\max(\epsilon,1-\beta_2^{\max(1,\mathrm{step})})},\quad \hat F_{bb}=\frac{F'_{bb}}{\max(\epsilon,1-\beta_2^{\max(1,\mathrm{step})})},\\ A&=\epsilon+\hat F_{ww},\quad C=\epsilon+\hat F_{bb},\quad \Delta=\max\!\left(10^{-12},AC-\hat F_{wb}^{\,2}\right),\\ r_w&=\eta\,\frac{\hat m_wC-\hat m_b\hat F_{wb}}{\Delta},\qquad r_b=\eta\,\frac{-\hat m_w\hat F_{wb}+\hat m_bA}{\Delta},\\ \kappa_w&=\tau\max(f_0,|w|),\qquad \kappa_b=\tau\max(f_0,|b|),\\ w'&= \begin{cases} w,&w-\operatorname{clip}(S(r_w),-\kappa_w,\kappa_w)\in\{\mathrm{NaN},\pm\infty\},\\ w-\operatorname{clip}(S(r_w),-\kappa_w,\kappa_w),&\text{otherwise}, \end{cases}\\ b'&= \begin{cases} b,&b-\operatorname{clip}(S(r_b),-\kappa_b,\kappa_b)\in\{\mathrm{NaN},\pm\infty\},\\ b-\operatorname{clip}(S(r_b),-\kappa_b,\kappa_b),&\text{otherwise}. \end{cases}\\ \operatorname{stack\_update\_sfd\_block2x2\_master} &=(w',b',m'_w,m'_b,F'_{ww},F'_{wb},F'_{bb}). \end{aligned} \]