Delgrange, F., Avni, G., Lukina, A., Schilling, C., Nowe, A., & Pérez, G. (2025). Composing Reinforcement Learning Policies, with Formal Guarantees AAAI Track. In Y. Vorobeychik, S. Das, & A. Nowe (Eds.), Proceedings of the 24th International Conference on Autonomous Agents and Multiagent Systems, AAMAS 2025 (Vol. Proceedings of the 24th International Conference on Autonomous Agents and Multiagent Systems, pp. 574-583). (Proceedings of the International Joint Conference on Autonomous Agents and Multiagent Systems, AAMAS). ACM. https://dl.acm.org/doi/10.5555/3709347.3743573

@inproceedings{b2904e3a7db449768a974eec81c59fe0,
title = "Composing Reinforcement Learning Policies, with Formal Guarantees AAAI Track",
abstract = "We propose a novel framework to controller design in environments with a two-level structure: a known high-level graph (''map'') in which each vertex is populated by a Markov decision process, called a ''room''. The framework ''separates concerns'' by using different design techniques for low- and high-level tasks. We apply reactive synthesis for high-level tasks: given a specification as a logical formula over the high-level graph and a collection of low-level policies obtained together with ''concise'' latent structures, we construct a ''planner'' that selects which low-level policy to apply in each room. We develop a reinforcement learning procedure to train low-level policies on latent structures, which unlike previous approaches, circumvents a model distillation step. We pair the policy with probably approximately correct guarantees on its performance and on the abstraction quality, and lift these guarantees to the high-level task. These formal guarantees are the main advantage of the framework. Other advantages include scalability (rooms are large and their dynamics are unknown) and reusability of low-level policies. We demonstrate feasibility in challenging case studies where an agent navigates environments with moving obstacles and visual inputs.",
keywords = "Reinforcement Learning, Reactive Synthesis, Formal Methods, Model Checking, Planning and Reasoning under Uncertainty, Representation Learning",
author = "Florent Delgrange and Guy Avni and Anna Lukina and Christian Schilling and Ann Nowe and Guillermo P{\'e}rez",
note = "Publisher Copyright: {\textcopyright} 2025 International Foundation for Autonomous Agents and Multiagent Systems (www.ifaamas.org).",
year = "2025",
month = jun,
day = "5",
language = "English",
volume = "Proceedings of the 24th International Conference on Autonomous Agents and Multiagent Systems",
series = "Proceedings of the International Joint Conference on Autonomous Agents and Multiagent Systems, AAMAS",
publisher = "ACM",
pages = "574--583",
editor = "Yevgeniy Vorobeychik and Sanmay Das and Ann Nowe",
booktitle = "Proceedings of the 24th International Conference on Autonomous Agents and Multiagent Systems, AAMAS 2025",
}