@article{Palomino-Resendiz2026, 
author = {Sergio Isai Palomino-Resendiz and César Ulises Solís-Cervantes and Luis Alberto Cantera-Cantera and Jorge de Jesús Morales-Mercado and Diego Alonso Flores-Hernández},
title = {Gradient Descent with Time-Decaying Regularization for Training Linear Neural Networks},
year = {2026},
journal = {Computer Modeling in Engineering & Sciences},
volume = {147},
number = {1},
pages = {26},
keywords = {Time-decaying regularization, gradient descent, single-layer linear neural network, online system identification, chaotic oscillator, embedded implementation},
url = {https://www.sciopen.com/article/10.32604/cmes.2026.077726},
doi = {10.32604/cmes.2026.077726},
abstract = {Many linear-in-parameters models arising in identification and control can be expressed as single-layer artificial neural networks (ANNs) with linear activation, enabling online learning via first-order optimization. In practice, however, standard gradient descent often exhibits slow convergence, large intermediate weights, and stagnation when the regressor data are ill-conditioned or computations are performed under finite precision. This paper proposes Gradient Descent with Time-Decaying Regularization (GD-TDR), a training algorithm that augments the quadratic loss with a regularization term whose weight decays exponentially in time. The proposed schedule enforces uniform strong convexity during early iterations, effectively mitigating neural-paralysis-like behavior associated with flat directions, while asymptotically vanishing so that the unregularized least-squares solution is recovered. A convergence theorem for GD-TDR is established and a concise pseudocode implementation is provided. Numerical and embedded experiments on an online identification problem of a Chua-type chaotic oscillator demonstrate that GD-TDR converges faster and avoids stagnation compared to standard gradient descent, without introducing the steady-state bias characteristic of fixed quadratic regularization.}
}