@article{Kanguzhin2026, 
author = {Baltabek Kanguzhin and Zhalgas Kaiyrbek and Zhamshid Khuzhakhmetov},
title = {Integral representation of the fundamental system of solutions for third order differential operators and completeness of the system of root functions of one irregular boundary value problem},
year = {2026},
journal = {AIMS Mathematics},
volume = {11},
number = {1},
pages = {2702-2721},
keywords = {differential operators, completeness, entire functions, integral equations, self-adjoint differential operators},
url = {https://www.sciopen.com/article/10.3934/math.2026109},
doi = {10.3934/math.2026109},
abstract = {This paper addresses two interrelated problems: the integral representation of solutions to third-order linear differential equations and the completeness of the root function system of the corresponding differential operator under irregular boundary conditions. In the first part, an integral representation for the fundamental system of solutions of a third-order differential equation with a complex spectral parameter is constructed. Unlike the classical approach by Marchenko, the obtained representations remain valid even when the coefficients are not holomorphic. The method is based on reducing the problem to Volterra integral equations of the second kind, which are solved using Picard's iterative method. Special representations are established for the initial terms of the iteration sequence, and a universal integral form is derived for the higher-order terms. The second part of the work focuses on a third-order differential operator on a finite interval with general irregular boundary conditions. The aim is to establish the completeness of the system of eigenfunctions and associated functions of this operator in the space        L          2      . To achieve this, properties of the characteristic determinant, its asymptotic behavior, and its relation to root functions are analyzed. It is proved that the root function system is complete even under boundary conditions that do not satisfy Birkhoff regularity. The results generalize known theorems for second-order operators and significantly extend the class of boundary conditions for which completeness holds. The proposed methods and results are of interest for the spectral theory of differential operators and the theory of transmutation operators and can be applied further to the study of inverse problems and problems with more general boundary conditions.}
}