@article{WANG2026, 
author = {Muhua WANG and Hui CHEN and Xiaoxiang ZHU and Tingzhao YU and Yu CHEN and Kuoyin WANG and Rui ZHAO and Zhimin LI and Bing XUE and Xinxin ZHANG},
title = {Evaluation of the capability of large language models in providing meteorological services in China},
year = {2026},
journal = {Acta Meteorologica Sinica},
volume = {84},
number = {5},
pages = {1004-1015},
keywords = {Large language models, Fenghe, Generative artificial intelligence, Meteorological services, Model evaluation},
url = {https://www.sciopen.com/article/10.11676/qxxb2026.20260046},
doi = {10.11676/qxxb2026.20260046},
abstract = {The rapid advancement of generative artificial intelligence is catalyzing an intelligent transformation centered on large language models (LLMs). Nevertheless, general-purpose LLMs typically encounter substantial limitations, including insufficient domain knowledge, low reliability in decision support, and elevated hallucination risks, when applied to specialized domains such as meteorological services. These challenges underscore the urgent need for a scientific and systematic evaluation framework tailored to domain specific LLM capabilities. An end-to-end evaluation paradigm for meteorological service LLMs is proposed, establishing a multidimensional capability assessment framework with quantitative metrics. Furthermore, a high quality benchmark dataset encompassing representative scenarios is developed. Systematic evaluations were conducted on the Fenghe model alongside nine mainstream LLMs. Results demonstrate that Fenghe significantly outperforms general purpose LLMs in critical competencies such as meteorological geographical understanding, interpretation of authoritative meteorological standards, and instruction driven meteorological content generation, thereby validating the efficacy of domain knowledge infusion and task oriented adaptation in enhancing specialized LLM performance. This work provides a methodological foundation for iterative optimization of artificial intelligence enabled meteorological services, model evaluation, and the establishment of industry application access mechanism.}
}