papers_database.json

{
  "rlhf_code": [
    {
      "title": "Execution-based Code Generation using Deep Reinforcement Learning",
      "authors": [
        "Parshin Shojaee",
        "Aneesh Jain",
        "Sindhu Tipirneni",
        "Chandan K. Reddy"
      ],
      "summary": "The utilization of programming language (PL) models, pre-trained on large-scale code corpora, as a means of automating software engineering processes has demonstrated considerable potential in streamlining various code generation tasks such as code completion, code translation, and program synthesis. However, current approaches mainly rely on supervised fine-tuning objectives borrowed from text generation, neglecting unique sequence-level characteristics of code, including but not limited to compilability as well as syntactic and functional correctness. To address this limitation, we propose PPOCoder, a new framework for code generation that synergistically combines pre-trained PL models with Proximal Policy Optimization (PPO) which is a widely used deep reinforcement learning technique. By utilizing non-differentiable feedback from code execution and structure alignment, PPOCoder seamlessly integrates external code-specific knowledge into the model optimization process. It's important to note that PPOCoder is a task-agnostic and model-agnostic framework that can be used across different code generation tasks and PLs. Extensive experiments on three code generation tasks demonstrate the effectiveness of our proposed approach compared to SOTA methods, achieving significant improvements in compilation success rates and functional correctness across different PLs.",
      "arxiv_id": "2301.13816v4",
      "published": "2023-01-31",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2301.13816v4",
      "pdf_url": "https://arxiv.org/pdf/2301.13816v4.pdf"
    },
    {
      "title": "Iterative Preference Learning from Human Feedback: Bridging Theory and Practice for RLHF under KL-Constraint",
      "authors": [
        "Wei Xiong",
        "Hanze Dong",
        "Chenlu Ye",
        "Ziqi Wang",
        "Han Zhong",
        "Heng Ji",
        "Nan Jiang",
        "Tong Zhang"
      ],
      "summary": "This paper studies the alignment process of generative models with Reinforcement Learning from Human Feedback (RLHF). We first identify the primary challenges of existing popular methods like offline PPO and offline DPO as lacking in strategical exploration of the environment. Then, to understand the mathematical principle of RLHF, we consider a standard mathematical formulation, the reverse-KL regularized contextual bandit for RLHF. Despite its widespread practical application, a rigorous theoretical analysis of this formulation remains open. We investigate its behavior in three distinct settings -- offline, online, and hybrid -- and propose efficient algorithms with finite-sample theoretical guarantees.   Moving towards practical applications, our framework, with a robust approximation of the information-theoretical policy improvement oracle, naturally gives rise to several novel RLHF algorithms. This includes an iterative version of the Direct Preference Optimization (DPO) algorithm for online settings, and a multi-step rejection sampling strategy for offline scenarios. Our empirical evaluations on real-world alignment experiment of large language model demonstrate that these proposed methods significantly surpass existing strong baselines, such as DPO and Rejection Sampling Optimization (RSO), showcasing the connections between solid theoretical foundations and their potent practical implementations.",
      "arxiv_id": "2312.11456v4",
      "published": "2023-12-18",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2312.11456v4",
      "pdf_url": "https://arxiv.org/pdf/2312.11456v4.pdf"
    },
    {
      "title": "Online Iterative Reinforcement Learning from Human Feedback with General Preference Model",
      "authors": [
        "Chenlu Ye",
        "Wei Xiong",
        "Yuheng Zhang",
        "Hanze Dong",
        "Nan Jiang",
        "Tong Zhang"
      ],
      "summary": "We investigate Reinforcement Learning from Human Feedback (RLHF) in the context of a general preference oracle. In particular, we do not assume the existence of a reward function and an oracle preference signal drawn from the Bradley-Terry model as most of the prior works do. We consider a standard mathematical formulation, the reverse-KL regularized minimax game between two LLMs for RLHF under general preference oracle. The learning objective of this formulation is to find a policy so that it is consistently preferred by the KL-regularized preference oracle over any competing LLMs. We show that this framework is strictly more general than the reward-based one, and propose sample-efficient algorithms for both the offline learning from a pre-collected preference dataset and online learning where we can query the preference oracle along the way of training. Empirical studies verify the effectiveness of the proposed framework.",
      "arxiv_id": "2402.07314v3",
      "published": "2024-02-11",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2402.07314v3",
      "pdf_url": "https://arxiv.org/pdf/2402.07314v3.pdf"
    },
    {
      "title": "Impoola: The Power of Average Pooling for Image-Based Deep Reinforcement Learning",
      "authors": [
        "Raphael Trumpp",
        "Ansgar Sch\u00e4fftlein",
        "Mirco Theile",
        "Marco Caccamo"
      ],
      "summary": "As image-based deep reinforcement learning tackles more challenging tasks, increasing model size has become an important factor in improving performance. Recent studies achieved this by focusing on the parameter efficiency of scaled networks, typically using Impala-CNN, a 15-layer ResNet-inspired network, as the image encoder. However, while Impala-CNN evidently outperforms older CNN architectures, potential advancements in network design for deep reinforcement learning-specific image encoders remain largely unexplored. We find that replacing the flattening of output feature maps in Impala-CNN with global average pooling leads to a notable performance improvement. This approach outperforms larger and more complex models in the Procgen Benchmark, particularly in terms of generalization. We call our proposed encoder model Impoola-CNN. A decrease in the network's translation sensitivity may be central to this improvement, as we observe the most significant gains in games without agent-centered observations. Our results demonstrate that network scaling is not just about increasing model size - efficient network design is also an essential factor. We make our code available at https://github.com/raphajaner/impoola.",
      "arxiv_id": "2503.05546v2",
      "published": "2025-03-07",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.05546v2",
      "pdf_url": "https://arxiv.org/pdf/2503.05546v2.pdf"
    },
    {
      "title": "DeepCodeSeek: Real-Time API Retrieval for Context-Aware Code Generation",
      "authors": [
        "Esakkivel Esakkiraja",
        "Denis Akhiyarov",
        "Aditya Shanmugham",
        "Chitra Ganapathy"
      ],
      "summary": "Current search techniques are limited to standard RAG query-document applications. In this paper, we propose a novel technique to expand the code and index for predicting the required APIs, directly enabling high-quality, end-to-end code generation for auto-completion and agentic AI applications. We address the problem of API leaks in current code-to-code benchmark datasets by introducing a new dataset built from real-world ServiceNow Script Includes that capture the challenge of unclear API usage intent in the code. Our evaluation metrics show that this method achieves 87.86% top-40 retrieval accuracy, allowing the critical context with APIs needed for successful downstream code generation. To enable real-time predictions, we develop a comprehensive post-training pipeline that optimizes a compact 0.6B reranker through synthetic dataset generation, supervised fine-tuning, and reinforcement learning. This approach enables our compact reranker to outperform a much larger 8B model while maintaining 2.5x reduced latency, effectively addressing the nuances of enterprise-specific code without the computational overhead of larger models.",
      "arxiv_id": "2509.25716v1",
      "published": "2025-09-30",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.IR"
      ],
      "url": "https://arxiv.org/abs/2509.25716v1",
      "pdf_url": "https://arxiv.org/pdf/2509.25716v1.pdf"
    },
    {
      "title": "ACE-RLHF: Automated Code Evaluation and Socratic Feedback Generation Tool using Large Language Models and Reinforcement Learning with Human Feedback",
      "authors": [
        "Tasnia Rahman",
        "Sathish A. P. Kumar",
        "Sumit Jha",
        "Arvind Ramanathan"
      ],
      "summary": "Automated Program Repair tools are developed for generating feedback and suggesting a repair method for erroneous code. State of the art (SOTA) code repair methods rely on data-driven approaches and often fail to deliver solution for complicated programming questions. To interpret the natural language of unprecedented programming problems, using Large Language Models (LLMs) for code-feedback generation is crucial. LLMs generate more comprehensible feedback than compiler-generated error messages, and Reinforcement Learning with Human Feedback (RLHF) further enhances quality by integrating human-in-the-loop which helps novice students to lean programming from scratch interactively. We are applying RLHF fine-tuning technique for an expected Socratic response such as a question with hint to solve the programming issue. We are proposing code feedback generation tool by fine-tuning LLM with RLHF, Automated Code Evaluation with RLHF (ACE-RLHF), combining two open-source LLM models with two different SOTA optimization techniques. The quality of feedback is evaluated on two benchmark datasets containing basic and competition-level programming questions where the later is proposed by us. We achieved 2-5% higher accuracy than RL-free SOTA techniques using Llama-3-7B-Proximal-policy optimization in automated evaluation and similar or slightly higher accuracy compared to reward model-free RL with AI Feedback (RLAIF). We achieved almost 40% higher accuracy with GPT-3.5 Best-of-n optimization while performing manual evaluation.",
      "arxiv_id": "2504.04657v1",
      "published": "2025-04-07",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.04657v1",
      "pdf_url": "https://arxiv.org/pdf/2504.04657v1.pdf"
    },
    {
      "title": "Deep Reinforcement Learning with Gradient Eligibility Traces",
      "authors": [
        "Esraa Elelimy",
        "Brett Daley",
        "Andrew Patterson",
        "Marlos C. Machado",
        "Adam White",
        "Martha White"
      ],
      "summary": "Achieving fast and stable off-policy learning in deep reinforcement learning (RL) is challenging. Most existing methods rely on semi-gradient temporal-difference (TD) methods for their simplicity and efficiency, but are consequently susceptible to divergence. While more principled approaches like Gradient TD (GTD) methods have strong convergence guarantees, they have rarely been used in deep RL. Recent work introduced the generalized Projected Bellman Error ($\\overline{\\text{PBE}}$), enabling GTD methods to work efficiently with nonlinear function approximation. However, this work is limited to one-step methods, which are slow at credit assignment and require a large number of samples. In this paper, we extend the generalized $\\overline{\\text{PBE}}$ objective to support multistep credit assignment based on the $\u03bb$-return and derive three gradient-based methods that optimize this new objective. We provide both a forward-view formulation compatible with experience replay and a backward-view formulation compatible with streaming algorithms. Finally, we evaluate the proposed algorithms and show that they outperform both PPO and StreamQ in MuJoCo and MinAtar environments, respectively. Code available at https://github.com/esraaelelimy/gtd\\_algos",
      "arxiv_id": "2507.09087v2",
      "published": "2025-07-12",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2507.09087v2",
      "pdf_url": "https://arxiv.org/pdf/2507.09087v2.pdf"
    },
    {
      "title": "Reinforcement Learning from Human Feedback with High-Confidence Safety Constraints",
      "authors": [
        "Yaswanth Chittepu",
        "Blossom Metevier",
        "Will Schwarzer",
        "Austin Hoag",
        "Scott Niekum",
        "Philip S. Thomas"
      ],
      "summary": "Existing approaches to language model alignment often treat safety as a tradeoff against helpfulness, which can lead to unacceptable responses in sensitive domains. To ensure reliable performance in such settings, we propose High-Confidence Safe Reinforcement Learning from Human Feedback (HC-RLHF), a method that provides high-confidence safety guarantees while maximizing helpfulness. Similar to previous methods, HC-RLHF explicitly decouples human preferences into helpfulness and harmlessness (safety), which are learned by training a reward model and a cost model, respectively. It then employs a two-step process to find safe solutions. In the first step, it optimizes the reward function under an intentionally pessimistic version of the cost constraint. In the second step, the trained model undergoes a safety test to verify whether its performance stays within an upper-confidence bound of the actual cost constraint. We provide a theoretical analysis of HC-RLHF, including proof that it will not return an unsafe solution with a probability greater than a user-specified threshold. For our empirical analysis, we apply HC-RLHF to align three different language models (Qwen2-1.5B, Qwen2.5-3B, and LLaMa3.2-3B) with human preferences. Our results demonstrate that HC-RLHF produces safe models with high probability and can improve harmlessness and helpfulness compared to previous methods.",
      "arxiv_id": "2506.08266v1",
      "published": "2025-06-09",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "stat.AP"
      ],
      "url": "https://arxiv.org/abs/2506.08266v1",
      "pdf_url": "https://arxiv.org/pdf/2506.08266v1.pdf"
    },
    {
      "title": "RLHF Deciphered: A Critical Analysis of Reinforcement Learning from Human Feedback for LLMs",
      "authors": [
        "Shreyas Chaudhari",
        "Pranjal Aggarwal",
        "Vishvak Murahari",
        "Tanmay Rajpurohit",
        "Ashwin Kalyan",
        "Karthik Narasimhan",
        "Ameet Deshpande",
        "Bruno Castro da Silva"
      ],
      "summary": "State-of-the-art large language models (LLMs) have become indispensable tools for various tasks. However, training LLMs to serve as effective assistants for humans requires careful consideration. A promising approach is reinforcement learning from human feedback (RLHF), which leverages human feedback to update the model in accordance with human preferences and mitigate issues like toxicity and hallucinations. Yet, an understanding of RLHF for LLMs is largely entangled with initial design choices that popularized the method and current research focuses on augmenting those choices rather than fundamentally improving the framework. In this paper, we analyze RLHF through the lens of reinforcement learning principles to develop an understanding of its fundamentals, dedicating substantial focus to the core component of RLHF -- the reward model. Our study investigates modeling choices, caveats of function approximation, and their implications on RLHF training algorithms, highlighting the underlying assumptions made about the expressivity of reward. Our analysis improves the understanding of the role of reward models and methods for their training, concurrently revealing limitations of the current methodology. We characterize these limitations, including incorrect generalization, model misspecification, and the sparsity of feedback, along with their impact on the performance of a language model. The discussion and analysis are substantiated by a categorical review of current literature, serving as a reference for researchers and practitioners to understand the challenges of RLHF and build upon existing efforts.",
      "arxiv_id": "2404.08555v2",
      "published": "2024-04-12",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2404.08555v2",
      "pdf_url": "https://arxiv.org/pdf/2404.08555v2.pdf"
    },
    {
      "title": "RLAIF vs. RLHF: Scaling Reinforcement Learning from Human Feedback with AI Feedback",
      "authors": [
        "Harrison Lee",
        "Samrat Phatale",
        "Hassan Mansoor",
        "Thomas Mesnard",
        "Johan Ferret",
        "Kellie Lu",
        "Colton Bishop",
        "Ethan Hall",
        "Victor Carbune",
        "Abhinav Rastogi",
        "Sushant Prakash"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has proven effective in aligning large language models (LLMs) with human preferences, but gathering high-quality preference labels is expensive. RL from AI Feedback (RLAIF), introduced in Bai et al., offers a promising alternative that trains the reward model (RM) on preferences generated by an off-the-shelf LLM. Across the tasks of summarization, helpful dialogue generation, and harmless dialogue generation, we show that RLAIF achieves comparable performance to RLHF. Furthermore, we take a step towards \"self-improvement\" by demonstrating that RLAIF can outperform a supervised fine-tuned baseline even when the AI labeler is the same size as the policy, or even the exact same checkpoint as the initial policy. Finally, we introduce direct-RLAIF (d-RLAIF) - a technique that circumvents RM training by obtaining rewards directly from an off-the-shelf LLM during RL, which achieves superior performance to canonical RLAIF. Our results suggest that RLAIF can achieve performance on-par with using human feedback, offering a potential solution to the scalability limitations of RLHF.",
      "arxiv_id": "2309.00267v3",
      "published": "2023-09-01",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2309.00267v3",
      "pdf_url": "https://arxiv.org/pdf/2309.00267v3.pdf"
    },
    {
      "title": "Automatic Unit Test Data Generation and Actor-Critic Reinforcement Learning for Code Synthesis",
      "authors": [
        "Philip John Gorinski",
        "Matthieu Zimmer",
        "Gerasimos Lampouras",
        "Derrick Goh Xin Deik",
        "Ignacio Iacobacci"
      ],
      "summary": "The advent of large pre-trained language models in the domain of Code Synthesis has shown remarkable performance on various benchmarks, treating the problem of Code Generation in a fashion similar to Natural Language Generation, trained with a Language Modelling (LM) objective. In addition, the property of programming language code being precisely evaluable with respect to its semantics -- through the use of Unit Tests to check its functional correctness -- lends itself to using Reinforcement Learning (RL) as a further training paradigm. Previous work has shown that RL can be applied as such to improve models' coding capabilities; however, such RL-based methods rely on a reward signal based on defined Unit Tests, which are much harder to obtain compared to the huge crawled code datasets used in LM objectives. In this work, we present a novel approach to automatically obtain data consisting of function signatures and associated Unit Tests, suitable for RL training of Code Synthesis models. We also introduce a straightforward, simple yet effective Actor-Critic RL training scheme and show that it, in conjunction with automatically generated training data, leads to improvement of a pre-trained code language model's performance by up to 9.9% improvement over the original underlying code synthesis LM, and up to 4.3% over RL-based models trained with standard PPO or CodeRL.",
      "arxiv_id": "2310.13669v1",
      "published": "2023-10-20",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2310.13669v1",
      "pdf_url": "https://arxiv.org/pdf/2310.13669v1.pdf"
    },
    {
      "title": "A Recipe for Unbounded Data Augmentation in Visual Reinforcement Learning",
      "authors": [
        "Abdulaziz Almuzairee",
        "Nicklas Hansen",
        "Henrik I. Christensen"
      ],
      "summary": "Q-learning algorithms are appealing for real-world applications due to their data-efficiency, but they are very prone to overfitting and training instabilities when trained from visual observations. Prior work, namely SVEA, finds that selective application of data augmentation can improve the visual generalization of RL agents without destabilizing training. We revisit its recipe for data augmentation, and find an assumption that limits its effectiveness to augmentations of a photometric nature. Addressing these limitations, we propose a generalized recipe, SADA, that works with wider varieties of augmentations. We benchmark its effectiveness on DMC-GB2 - our proposed extension of the popular DMControl Generalization Benchmark - as well as tasks from Meta-World and the Distracting Control Suite, and find that our method, SADA, greatly improves training stability and generalization of RL agents across a diverse set of augmentations. For visualizations, code and benchmark: see https://aalmuzairee.github.io/SADA/",
      "arxiv_id": "2405.17416v2",
      "published": "2024-05-27",
      "categories": [
        "cs.LG",
        "cs.CV",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2405.17416v2",
      "pdf_url": "https://arxiv.org/pdf/2405.17416v2.pdf"
    },
    {
      "title": "Policy-labeled Preference Learning: Is Preference Enough for RLHF?",
      "authors": [
        "Taehyun Cho",
        "Seokhun Ju",
        "Seungyub Han",
        "Dohyeong Kim",
        "Kyungjae Lee",
        "Jungwoo Lee"
      ],
      "summary": "To design rewards that align with human goals, Reinforcement Learning from Human Feedback (RLHF) has emerged as a prominent technique for learning reward functions from human preferences and optimizing policies via reinforcement learning algorithms. However, existing RLHF methods often misinterpret trajectories as being generated by an optimal policy, causing inaccurate likelihood estimation and suboptimal learning. Inspired by Direct Preference Optimization framework which directly learns optimal policy without explicit reward, we propose policy-labeled preference learning (PPL), to resolve likelihood mismatch issues by modeling human preferences with regret, which reflects behavior policy information. We also provide a contrastive KL regularization, derived from regret-based principles, to enhance RLHF in sequential decision making. Experiments in high-dimensional continuous control tasks demonstrate PPL's significant improvements in offline RLHF performance and its effectiveness in online settings.",
      "arxiv_id": "2505.06273v2",
      "published": "2025-05-06",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.06273v2",
      "pdf_url": "https://arxiv.org/pdf/2505.06273v2.pdf"
    },
    {
      "title": "Dreaming of Many Worlds: Learning Contextual World Models Aids Zero-Shot Generalization",
      "authors": [
        "Sai Prasanna",
        "Karim Farid",
        "Raghu Rajan",
        "Andr\u00e9 Biedenkapp"
      ],
      "summary": "Zero-shot generalization (ZSG) to unseen dynamics is a major challenge for creating generally capable embodied agents. To address the broader challenge, we start with the simpler setting of contextual reinforcement learning (cRL), assuming observability of the context values that parameterize the variation in the system's dynamics, such as the mass or dimensions of a robot, without making further simplifying assumptions about the observability of the Markovian state. Toward the goal of ZSG to unseen variation in context, we propose the contextual recurrent state-space model (cRSSM), which introduces changes to the world model of Dreamer (v3) (Hafner et al., 2023). This allows the world model to incorporate context for inferring latent Markovian states from the observations and modeling the latent dynamics. Our approach is evaluated on two tasks from the CARL benchmark suite, which is tailored to study contextual RL. Our experiments show that such systematic incorporation of the context improves the ZSG of the policies trained on the \"dreams\" of the world model. We further find qualitatively that our approach allows Dreamer to disentangle the latent state from context, allowing it to extrapolate its dreams to the many worlds of unseen contexts. The code for all our experiments is available at https://github.com/sai-prasanna/dreaming_of_many_worlds.",
      "arxiv_id": "2403.10967v2",
      "published": "2024-03-16",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2403.10967v2",
      "pdf_url": "https://arxiv.org/pdf/2403.10967v2.pdf"
    },
    {
      "title": "Uni-RLHF: Universal Platform and Benchmark Suite for Reinforcement Learning with Diverse Human Feedback",
      "authors": [
        "Yifu Yuan",
        "Jianye Hao",
        "Yi Ma",
        "Zibin Dong",
        "Hebin Liang",
        "Jinyi Liu",
        "Zhixin Feng",
        "Kai Zhao",
        "Yan Zheng"
      ],
      "summary": "Reinforcement Learning with Human Feedback (RLHF) has received significant attention for performing tasks without the need for costly manual reward design by aligning human preferences. It is crucial to consider diverse human feedback types and various learning methods in different environments. However, quantifying progress in RLHF with diverse feedback is challenging due to the lack of standardized annotation platforms and widely used unified benchmarks. To bridge this gap, we introduce Uni-RLHF, a comprehensive system implementation tailored for RLHF. It aims to provide a complete workflow from real human feedback, fostering progress in the development of practical problems. Uni-RLHF contains three packages: 1) a universal multi-feedback annotation platform, 2) large-scale crowdsourced feedback datasets, and 3) modular offline RLHF baseline implementations. Uni-RLHF develops a user-friendly annotation interface tailored to various feedback types, compatible with a wide range of mainstream RL environments. We then establish a systematic pipeline of crowdsourced annotations, resulting in large-scale annotated datasets comprising more than 15 million steps across 30+ popular tasks. Through extensive experiments, the results in the collected datasets demonstrate competitive performance compared to those from well-designed manual rewards. We evaluate various design choices and offer insights into their strengths and potential areas of improvement. We wish to build valuable open-source platforms, datasets, and baselines to facilitate the development of more robust and reliable RLHF solutions based on realistic human feedback. The website is available at https://uni-rlhf.github.io/.",
      "arxiv_id": "2402.02423v2",
      "published": "2024-02-04",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.HC",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2402.02423v2",
      "pdf_url": "https://arxiv.org/pdf/2402.02423v2.pdf"
    },
    {
      "title": "On a Connection Between Imitation Learning and RLHF",
      "authors": [
        "Teng Xiao",
        "Yige Yuan",
        "Mingxiao Li",
        "Zhengyu Chen",
        "Vasant G Honavar"
      ],
      "summary": "This work studies the alignment of large language models with preference data from an imitation learning perspective. We establish a close theoretical connection between reinforcement learning from human feedback RLHF and imitation learning (IL), revealing that RLHF implicitly performs imitation learning on the preference data distribution. Building on this connection, we propose DIL, a principled framework that directly optimizes the imitation learning objective. DIL provides a unified imitation learning perspective on alignment, encompassing existing alignment algorithms as special cases while naturally introducing new variants. By bridging IL and RLHF, DIL offers new insights into alignment with RLHF. Extensive experiments demonstrate that DIL outperforms existing methods on various challenging benchmarks.",
      "arxiv_id": "2503.05079v1",
      "published": "2025-03-07",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.05079v1",
      "pdf_url": "https://arxiv.org/pdf/2503.05079v1.pdf"
    },
    {
      "title": "CodeRL: Mastering Code Generation through Pretrained Models and Deep Reinforcement Learning",
      "authors": [
        "Hung Le",
        "Yue Wang",
        "Akhilesh Deepak Gotmare",
        "Silvio Savarese",
        "Steven C. H. Hoi"
      ],
      "summary": "Program synthesis or code generation aims to generate a program that satisfies a problem specification. Recent approaches using large-scale pretrained language models (LMs) have shown promising results, yet they have some critical limitations. In particular, they often follow a standard supervised fine-tuning procedure to train a code generation model only from the pairs of natural-language problem descriptions and ground-truth programs. Such paradigm largely ignores some important but potentially useful signals in the problem specification such as unit tests, which thus often results in poor performance when solving complex unseen coding tasks. To address the limitations, we propose \"CodeRL\", a new framework for program synthesis tasks through pretrained LMs and deep reinforcement learning (RL). Specifically, during training, we treat the code-generating LM as an actor network, and introduce a critic network that is trained to predict the functional correctness of generated programs and provide dense feedback signals to the actor. During inference, we introduce a new generation procedure with a critical sampling strategy that allows a model to automatically regenerate programs based on feedback from example unit tests and critic scores. For the model backbones, we extended the encoder-decoder architecture of CodeT5 with enhanced learning objectives, larger model sizes, and better pretraining data. Our method not only achieves new SOTA results on the challenging APPS benchmark, but also shows strong zero-shot transfer capability with new SOTA results on the simpler MBPP benchmark.",
      "arxiv_id": "2207.01780v3",
      "published": "2022-07-05",
      "categories": [
        "cs.LG",
        "cs.CL",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2207.01780v3",
      "pdf_url": "https://arxiv.org/pdf/2207.01780v3.pdf"
    },
    {
      "title": "RLHF Workflow: From Reward Modeling to Online RLHF",
      "authors": [
        "Hanze Dong",
        "Wei Xiong",
        "Bo Pang",
        "Haoxiang Wang",
        "Han Zhao",
        "Yingbo Zhou",
        "Nan Jiang",
        "Doyen Sahoo",
        "Caiming Xiong",
        "Tong Zhang"
      ],
      "summary": "We present the workflow of Online Iterative Reinforcement Learning from Human Feedback (RLHF) in this technical report, which is widely reported to outperform its offline counterpart by a large margin in the recent large language model (LLM) literature. However, existing open-source RLHF projects are still largely confined to the offline learning setting. In this technical report, we aim to fill in this gap and provide a detailed recipe that is easy to reproduce for online iterative RLHF. In particular, since online human feedback is usually infeasible for open-source communities with limited resources, we start by constructing preference models using a diverse set of open-source datasets and use the constructed proxy preference model to approximate human feedback. Then, we discuss the theoretical insights and algorithmic principles behind online iterative RLHF, followed by a detailed practical implementation. Our trained LLM achieves impressive performance on LLM chatbot benchmarks, including AlpacaEval-2, Arena-Hard, and MT-Bench, as well as other academic benchmarks such as HumanEval and TruthfulQA. We have shown that supervised fine-tuning (SFT) and iterative RLHF can obtain state-of-the-art performance with fully open-source datasets. Further, we have made our models, curated datasets, and comprehensive step-by-step code guidebooks publicly available. Please refer to https://github.com/RLHFlow/RLHF-Reward-Modeling and https://github.com/RLHFlow/Online-RLHF for more detailed information.",
      "arxiv_id": "2405.07863v3",
      "published": "2024-05-13",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2405.07863v3",
      "pdf_url": "https://arxiv.org/pdf/2405.07863v3.pdf"
    },
    {
      "title": "Principled Penalty-based Methods for Bilevel Reinforcement Learning and RLHF",
      "authors": [
        "Han Shen",
        "Zhuoran Yang",
        "Tianyi Chen"
      ],
      "summary": "Bilevel optimization has been recently applied to many machine learning tasks. However, their applications have been restricted to the supervised learning setting, where static objective functions with benign structures are considered. But bilevel problems such as incentive design, inverse reinforcement learning (RL), and RL from human feedback (RLHF) are often modeled as dynamic objective functions that go beyond the simple static objective structures, which pose significant challenges of using existing bilevel solutions. To tackle this new class of bilevel problems, we introduce the first principled algorithmic framework for solving bilevel RL problems through the lens of penalty formulation. We provide theoretical studies of the problem landscape and its penalty-based (policy) gradient algorithms. We demonstrate the effectiveness of our algorithms via simulations in the Stackelberg Markov game, RL from human feedback and incentive design.",
      "arxiv_id": "2402.06886v3",
      "published": "2024-02-10",
      "categories": [
        "cs.LG",
        "math.OC",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2402.06886v3",
      "pdf_url": "https://arxiv.org/pdf/2402.06886v3.pdf"
    },
    {
      "title": "Reinforcement Learning in the Era of LLMs: What is Essential? What is needed? An RL Perspective on RLHF, Prompting, and Beyond",
      "authors": [
        "Hao Sun"
      ],
      "summary": "Recent advancements in Large Language Models (LLMs) have garnered wide attention and led to successful products such as ChatGPT and GPT-4. Their proficiency in adhering to instructions and delivering harmless, helpful, and honest (3H) responses can largely be attributed to the technique of Reinforcement Learning from Human Feedback (RLHF). In this paper, we aim to link the research in conventional RL to RL techniques used in LLM research. Demystify this technique by discussing why, when, and how RL excels. Furthermore, we explore potential future avenues that could either benefit from or contribute to RLHF research.   Highlighted Takeaways:   1. RLHF is Online Inverse RL with Offline Demonstration Data.   2. RLHF $>$ SFT because Imitation Learning (and Inverse RL) $>$ Behavior Cloning (BC) by alleviating the problem of compounding error.   3. The RM step in RLHF generates a proxy of the expensive human feedback, such an insight can be generalized to other LLM tasks such as prompting evaluation and optimization where feedback is also expensive.   4. The policy learning in RLHF is more challenging than conventional problems studied in IRL due to their high action dimensionality and feedback sparsity.   5. The main superiority of PPO over off-policy value-based methods is its stability gained from (almost) on-policy data and conservative policy updates.",
      "arxiv_id": "2310.06147v1",
      "published": "2023-10-09",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2310.06147v1",
      "pdf_url": "https://arxiv.org/pdf/2310.06147v1.pdf"
    },
    {
      "title": "Safe RLHF: Safe Reinforcement Learning from Human Feedback",
      "authors": [
        "Josef Dai",
        "Xuehai Pan",
        "Ruiyang Sun",
        "Jiaming Ji",
        "Xinbo Xu",
        "Mickel Liu",
        "Yizhou Wang",
        "Yaodong Yang"
      ],
      "summary": "With the development of large language models (LLMs), striking a balance between the performance and safety of AI systems has never been more critical. However, the inherent tension between the objectives of helpfulness and harmlessness presents a significant challenge during LLM training. To address this issue, we propose Safe Reinforcement Learning from Human Feedback (Safe RLHF), a novel algorithm for human value alignment. Safe RLHF explicitly decouples human preferences regarding helpfulness and harmlessness, effectively avoiding the crowdworkers' confusion about the tension and allowing us to train separate reward and cost models. We formalize the safety concern of LLMs as an optimization task of maximizing the reward function while satisfying specified cost constraints. Leveraging the Lagrangian method to solve this constrained problem, Safe RLHF dynamically adjusts the balance between the two objectives during fine-tuning. Through a three-round fine-tuning using Safe RLHF, we demonstrate a superior ability to mitigate harmful responses while enhancing model performance compared to existing value-aligned algorithms. Experimentally, we fine-tuned the Alpaca-7B using Safe RLHF and aligned it with collected human preferences, significantly improving its helpfulness and harmlessness according to human evaluations.",
      "arxiv_id": "2310.12773v1",
      "published": "2023-10-19",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2310.12773v1",
      "pdf_url": "https://arxiv.org/pdf/2310.12773v1.pdf"
    },
    {
      "title": "Fly, Fail, Fix: Iterative Game Repair with Reinforcement Learning and Large Multimodal Models",
      "authors": [
        "Alex Zook",
        "Josef Spjut",
        "Jonathan Tremblay"
      ],
      "summary": "Game design hinges on understanding how static rules and content translate into dynamic player behavior - something modern generative systems that inspect only a game's code or assets struggle to capture. We present an automated design iteration framework that closes this gap by pairing a reinforcement learning (RL) agent, which playtests the game, with a large multimodal model (LMM), which revises the game based on what the agent does. In each loop the RL player completes several episodes, producing (i) numerical play metrics and/or (ii) a compact image strip summarising recent video frames. The LMM designer receives a gameplay goal and the current game configuration, analyses the play traces, and edits the configuration to steer future behaviour toward the goal. We demonstrate results that LMMs can reason over behavioral traces supplied by RL agents to iteratively refine game mechanics, pointing toward practical, scalable tools for AI-assisted game design.",
      "arxiv_id": "2507.12666v1",
      "published": "2025-07-16",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2507.12666v1",
      "pdf_url": "https://arxiv.org/pdf/2507.12666v1.pdf"
    },
    {
      "title": "DPO Meets PPO: Reinforced Token Optimization for RLHF",
      "authors": [
        "Han Zhong",
        "Zikang Shan",
        "Guhao Feng",
        "Wei Xiong",
        "Xinle Cheng",
        "Li Zhao",
        "Di He",
        "Jiang Bian",
        "Liwei Wang"
      ],
      "summary": "In the classical Reinforcement Learning from Human Feedback (RLHF) framework, Proximal Policy Optimization (PPO) is employed to learn from sparse, sentence-level rewards -- a challenging scenario in traditional deep reinforcement learning. Despite the great successes of PPO in the alignment of large language models, its open-source implementation is still largely sub-optimal. To address these issues, we introduce a framework that models RLHF problems as a Markov decision process (MDP), enabling the capture of fine-grained token-wise information. Under this framework, we introduce an algorithm Reinforced Token Optimization (\\texttt{RTO}), which learns the token-wise reward function from preference data and performs policy optimization based on this learned token-wise reward signal. Theoretically, \\texttt{RTO} is proven to have the capability of finding the near-optimal policy sample-efficiently. For its practical implementation, \\texttt{RTO} innovatively integrates Direct Preference Optimization (DPO) and PPO. DPO, originally derived from sparse sentence rewards, surprisingly provides us with a token-wise characterization of response quality, which is seamlessly incorporated into our subsequent PPO training stage. Extensive experiments demonstrate that \\texttt{RTO} performs better than PPO and other direct preference learning algorithms. In particular, RTO outperforms PPO by 7.5 points on the AlpacaEval 2 benchmark and by 4.1 points on Arena-Hard. Our code and models are available at \\href{https://github.com/zkshan2002/RTO}{https://github.com/zkshan2002/RTO}.",
      "arxiv_id": "2404.18922v4",
      "published": "2024-04-29",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2404.18922v4",
      "pdf_url": "https://arxiv.org/pdf/2404.18922v4.pdf"
    },
    {
      "title": "The Hidden Link Between RLHF and Contrastive Learning",
      "authors": [
        "Xufei Lv",
        "Kehai Chen",
        "Haoyuan Sun",
        "Xuefeng Bai",
        "Min Zhang",
        "Houde Liu",
        "Kehai Chen"
      ],
      "summary": "Alignment of large language models (LLMs) with human values has recently garnered significant attention, with prominent examples including the canonical yet costly Reinforcement Learning from Human Feedback (RLHF) and the simple Direct Preference Optimization (DPO). In this work, we demonstrate that both RLHF and DPO can be interpreted from the perspective of mutual information (MI) maximization, uncovering a profound connection to contrastive learning. Within this framework, both RLHF and DPO can be interpreted as methods that performing contrastive learning based on the positive and negative samples derived from base model, leveraging the Donsker-Varadhan (DV) lower bound on MI (equivalently, the MINE estimator). Such paradigm further illuminates why RLHF may not intrinsically incentivize reasoning capacities in LLMs beyond what is already present in the base model. Building on the perspective, we replace the DV/MINE bound with the Jensen-Shannon (JS) MI estimator and propose the Mutual Information Optimization (MIO). Comprehensive theoretical analysis and extensive empirical evaluations demonstrate that MIO mitigates the late-stage decline in chosen-likelihood observed in DPO, achieving competitive or superior performance across various challenging reasoning and mathematical benchmarks.",
      "arxiv_id": "2506.22578v2",
      "published": "2025-06-27",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2506.22578v2",
      "pdf_url": "https://arxiv.org/pdf/2506.22578v2.pdf"
    },
    {
      "title": "Process-Supervised Reinforcement Learning for Code Generation",
      "authors": [
        "Yufan Ye",
        "Ting Zhang",
        "Wenbin Jiang",
        "Hua Huang"
      ],
      "summary": "Existing reinforcement learning strategies based on outcome supervision have proven effective in enhancing the performance of large language models(LLMs) for code generation. While reinforcement learning based on process supervision has shown great promise in handling multi-step reasoning tasks, its effectiveness in code generation remains largely underexplored and underjustified. The primary obstacle stems from the resource-intensive nature of constructing high-quality process-supervised data, which demands substantial human expertise and computational resources. In response to this challenge, we propose a \"statement mutation/refactoring-compile and execution verification\" strategy: mutating and refactoring code line-by-line through a teacher model, and utilizing compiler execution results to automatically label each line, resulting in line-by-line process-supervised data, which is pivotal for training a process-supervised reward model. The trained reward model is then integrated into the PRLCoder framework, followed by experimental validation on several benchmarks. Experimental results demonstrate that process-supervised reinforcement learning significantly surpasses methods relying solely on outcome supervision. Notably, in tackling complex code generation tasks, process-supervised reinforcement learning shows a clear advantage, ensuring both the integrity of the code generation process and the correctness of the generation results.",
      "arxiv_id": "2502.01715v1",
      "published": "2025-02-03",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2502.01715v1",
      "pdf_url": "https://arxiv.org/pdf/2502.01715v1.pdf"
    },
    {
      "title": "Reinforcement Learning under State and Outcome Uncertainty: A Foundational Distributional Perspective",
      "authors": [
        "Larry Preuett",
        "Qiuyi Zhang",
        "Muhammad Aurangzeb Ahmad"
      ],
      "summary": "In many real-world planning tasks, agents must tackle uncertainty about the environment's state and variability in the outcomes of any chosen policy. We address both forms of uncertainty as a first step toward safer algorithms in partially observable settings. Specifically, we extend Distributional Reinforcement Learning (DistRL)-which models the entire return distribution for fully observable domains-to Partially Observable Markov Decision Processes (POMDPs), allowing an agent to learn the distribution of returns for each conditional plan. Concretely, we introduce new distributional Bellman operators for partial observability and prove their convergence under the supremum p-Wasserstein metric. We also propose a finite representation of these return distributions via psi-vectors, generalizing the classical alpha-vectors in POMDP solvers. Building on this, we develop Distributional Point-Based Value Iteration (DPBVI), which integrates psi-vectors into a standard point-based backup procedure-bridging DistRL and POMDP planning. By tracking return distributions, DPBVI naturally enables risk-sensitive control in domains where rare, high-impact events must be carefully managed. We provide source code to foster further research in robust decision-making under partial observability.",
      "arxiv_id": "2505.06518v2",
      "published": "2025-05-10",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.06518v2",
      "pdf_url": "https://arxiv.org/pdf/2505.06518v2.pdf"
    },
    {
      "title": "An Investigation of Offline Reinforcement Learning in Factorisable Action Spaces",
      "authors": [
        "Alex Beeson",
        "David Ireland",
        "Giovanni Montana"
      ],
      "summary": "Expanding reinforcement learning (RL) to offline domains generates promising prospects, particularly in sectors where data collection poses substantial challenges or risks. Pivotal to the success of transferring RL offline is mitigating overestimation bias in value estimates for state-action pairs absent from data. Whilst numerous approaches have been proposed in recent years, these tend to focus primarily on continuous or small-scale discrete action spaces. Factorised discrete action spaces, on the other hand, have received relatively little attention, despite many real-world problems naturally having factorisable actions. In this work, we undertake a formative investigation into offline reinforcement learning in factorisable action spaces. Using value-decomposition as formulated in DecQN as a foundation, we present the case for a factorised approach and conduct an extensive empirical evaluation of several offline techniques adapted to the factorised setting. In the absence of established benchmarks, we introduce a suite of our own comprising datasets of varying quality and task complexity. Advocating for reproducible research and innovation, we make all datasets available for public use alongside our code base.",
      "arxiv_id": "2411.11088v1",
      "published": "2024-11-17",
      "categories": [
        "stat.ML",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2411.11088v1",
      "pdf_url": "https://arxiv.org/pdf/2411.11088v1.pdf"
    },
    {
      "title": "On Generalization Across Environments In Multi-Objective Reinforcement Learning",
      "authors": [
        "Jayden Teoh",
        "Pradeep Varakantham",
        "Peter Vamplew"
      ],
      "summary": "Real-world sequential decision-making tasks often require balancing trade-offs between multiple conflicting objectives, making Multi-Objective Reinforcement Learning (MORL) an increasingly prominent field of research. Despite recent advances, existing MORL literature has narrowly focused on performance within static environments, neglecting the importance of generalizing across diverse settings. Conversely, existing research on generalization in RL has always assumed scalar rewards, overlooking the inherent multi-objectivity of real-world problems. Generalization in the multi-objective context is fundamentally more challenging, as it requires learning a Pareto set of policies addressing varying preferences across multiple objectives. In this paper, we formalize the concept of generalization in MORL and how it can be evaluated. We then contribute a novel benchmark featuring diverse multi-objective domains with parameterized environment configurations to facilitate future studies in this area. Our baseline evaluations of state-of-the-art MORL algorithms on this benchmark reveals limited generalization capabilities, suggesting significant room for improvement. Our empirical findings also expose limitations in the expressivity of scalar rewards, emphasizing the need for multi-objective specifications to achieve effective generalization. We further analyzed the algorithmic complexities within current MORL approaches that could impede the transfer in performance from the single- to multiple-environment settings. This work fills a critical gap and lays the groundwork for future research that brings together two key areas in reinforcement learning: solving multi-objective decision-making problems and generalizing across diverse environments. We make our code available at https://github.com/JaydenTeoh/MORL-Generalization.",
      "arxiv_id": "2503.00799v2",
      "published": "2025-03-02",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.00799v2",
      "pdf_url": "https://arxiv.org/pdf/2503.00799v2.pdf"
    },
    {
      "title": "Learning to Generate Unit Test via Adversarial Reinforcement Learning",
      "authors": [
        "Dongjun Lee",
        "Changho Hwang",
        "Kimin Lee"
      ],
      "summary": "Unit testing is a core practice in programming, enabling systematic evaluation of programs produced by human developers or large language models (LLMs). Given the challenges in writing comprehensive unit tests, LLMs have been employed to automate test generation, yet methods for training LLMs to produce high-quality tests remain underexplored. In this work, we propose UTRL, a novel reinforcement learning framework that trains an LLM to generate high-quality unit tests given a programming instruction. Our key idea is to iteratively train two LLMs, the unit test generator and the code generator, in an adversarial manner via reinforcement learning. The unit test generator is trained to maximize a discrimination reward, which reflects its ability to produce tests that expose faults in the code generator's solutions, and the code generator is trained to maximize a code reward, which reflects its ability to produce solutions that pass the unit tests generated by the test generator. In our experiments, we demonstrate that unit tests generated by Qwen3-4B trained via UTRL show higher quality compared to unit tests generated by the same model trained via supervised fine-tuning on human-written ground-truth unit tests, yielding code evaluations that more closely align with those induced by the ground-truth tests. Moreover, Qwen3-4B trained with UTRL outperforms frontier models such as GPT-4.1 in generating high-quality unit tests, highlighting the effectiveness of UTRL in training LLMs for this task.",
      "arxiv_id": "2508.21107v2",
      "published": "2025-08-28",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.21107v2",
      "pdf_url": "https://arxiv.org/pdf/2508.21107v2.pdf"
    },
    {
      "title": "RLHF-Blender: A Configurable Interactive Interface for Learning from Diverse Human Feedback",
      "authors": [
        "Yannick Metz",
        "David Lindner",
        "Rapha\u00ebl Baur",
        "Daniel Keim",
        "Mennatallah El-Assady"
      ],
      "summary": "To use reinforcement learning from human feedback (RLHF) in practical applications, it is crucial to learn reward models from diverse sources of human feedback and to consider human factors involved in providing feedback of different types. However, the systematic study of learning from diverse types of feedback is held back by limited standardized tooling available to researchers. To bridge this gap, we propose RLHF-Blender, a configurable, interactive interface for learning from human feedback. RLHF-Blender provides a modular experimentation framework and implementation that enables researchers to systematically investigate the properties and qualities of human feedback for reward learning. The system facilitates the exploration of various feedback types, including demonstrations, rankings, comparisons, and natural language instructions, as well as studies considering the impact of human factors on their effectiveness. We discuss a set of concrete research opportunities enabled by RLHF-Blender. More information is available at https://rlhfblender.info/.",
      "arxiv_id": "2308.04332v1",
      "published": "2023-08-08",
      "categories": [
        "cs.LG",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2308.04332v1",
      "pdf_url": "https://arxiv.org/pdf/2308.04332v1.pdf"
    },
    {
      "title": "BLIP-FusePPO: A Vision-Language Deep Reinforcement Learning Framework for Lane Keeping in Autonomous Vehicles",
      "authors": [
        "Seyed Ahmad Hosseini Miangoleh",
        "Amin Jalal Aghdasian",
        "Farzaneh Abdollahi"
      ],
      "summary": "In this paper, we propose Bootstrapped Language-Image Pretraining-driven Fused State Representation in Proximal Policy Optimization (BLIP-FusePPO), a novel multimodal reinforcement learning (RL) framework for autonomous lane-keeping (LK), in which semantic embeddings generated by a vision-language model (VLM) are directly fused with geometric states, LiDAR observations, and Proportional-Integral-Derivative-based (PID) control feedback within the agent observation space. The proposed method lets the agent learn driving rules that are aware of their surroundings and easy to understand by combining high-level scene understanding from the VLM with low-level control and spatial signals. Our architecture brings together semantic, geometric, and control-aware representations to make policy learning more robust. A hybrid reward function that includes semantic alignment, LK accuracy, obstacle avoidance, and speed regulation helps learning to be more efficient and generalizable. Our method is different from the approaches that only use semantic models to shape rewards. Instead, it directly embeds semantic features into the state representation. This cuts down on expensive runtime inference and makes sure that semantic guidance is always available. The simulation results show that the proposed model is better at LK stability and adaptability than the best vision-based and multimodal RL baselines in a wide range of difficult driving situations. We make our code publicly available.",
      "arxiv_id": "2510.22370v1",
      "published": "2025-10-25",
      "categories": [
        "cs.RO",
        "cs.AI",
        "cs.CV",
        "cs.LG",
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2510.22370v1",
      "pdf_url": "https://arxiv.org/pdf/2510.22370v1.pdf"
    },
    {
      "title": "Training a Helpful and Harmless Assistant with Reinforcement Learning from Human Feedback",
      "authors": [
        "Yuntao Bai",
        "Andy Jones",
        "Kamal Ndousse",
        "Amanda Askell",
        "Anna Chen",
        "Nova DasSarma",
        "Dawn Drain",
        "Stanislav Fort",
        "Deep Ganguli",
        "Tom Henighan",
        "Nicholas Joseph",
        "Saurav Kadavath",
        "Jackson Kernion",
        "Tom Conerly",
        "Sheer El-Showk",
        "Nelson Elhage",
        "Zac Hatfield-Dodds",
        "Danny Hernandez",
        "Tristan Hume",
        "Scott Johnston",
        "Shauna Kravec",
        "Liane Lovitt",
        "Neel Nanda",
        "Catherine Olsson",
        "Dario Amodei",
        "Tom Brown",
        "Jack Clark",
        "Sam McCandlish",
        "Chris Olah",
        "Ben Mann",
        "Jared Kaplan"
      ],
      "summary": "We apply preference modeling and reinforcement learning from human feedback (RLHF) to finetune language models to act as helpful and harmless assistants. We find this alignment training improves performance on almost all NLP evaluations, and is fully compatible with training for specialized skills such as python coding and summarization. We explore an iterated online mode of training, where preference models and RL policies are updated on a weekly cadence with fresh human feedback data, efficiently improving our datasets and models. Finally, we investigate the robustness of RLHF training, and identify a roughly linear relation between the RL reward and the square root of the KL divergence between the policy and its initialization. Alongside our main results, we perform peripheral analyses on calibration, competing objectives, and the use of OOD detection, compare our models with human writers, and provide samples from our models using prompts appearing in recent related work.",
      "arxiv_id": "2204.05862v1",
      "published": "2022-04-12",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2204.05862v1",
      "pdf_url": "https://arxiv.org/pdf/2204.05862v1.pdf"
    },
    {
      "title": "Reinforcement Learning and Data-Generation for Syntax-Guided Synthesis",
      "authors": [
        "Julian Parsert",
        "Elizabeth Polgreen"
      ],
      "summary": "Program synthesis is the task of automatically generating code based on a specification. In Syntax-Guided Synthesis (SyGuS) this specification is a combination of a syntactic template and a logical formula, and the result is guaranteed to satisfy both.   We present a reinforcement-learning guided algorithm for SyGuS which uses Monte-Carlo Tree Search (MCTS) to search the space of candidate solutions. Our algorithm learns policy and value functions which, combined with the upper confidence bound for trees, allow it to balance exploration and exploitation. A common challenge in applying machine learning approaches to syntax-guided synthesis is the scarcity of training data. To address this, we present a method for automatically generating training data for SyGuS based on anti-unification of existing first-order satisfiability problems, which we use to train our MCTS policy. We implement and evaluate this setup and demonstrate that learned policy and value improve the synthesis performance over a baseline by over 26 percentage points in the training and testing sets. Our tool outperforms state-of-the-art tool cvc5 on the training set and performs comparably in terms of the total number of problems solved on the testing set (solving 23% of the benchmarks on which cvc5 fails). We make our data set publicly available, to enable further application of machine learning methods to the SyGuS problem.",
      "arxiv_id": "2307.09564v2",
      "published": "2023-07-13",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2307.09564v2",
      "pdf_url": "https://arxiv.org/pdf/2307.09564v2.pdf"
    },
    {
      "title": "CosmoCore Affective Dream-Replay Reinforcement Learning for Code Generation",
      "authors": [
        "Santhosh Kumar Ravindran"
      ],
      "summary": "We introduce CosmoCore, a neuroscience-inspired reinforcement learning (RL) architecture that integrates affective signals to enhance code generation in large language models (LLMs). Motivated by human and animal learning where embarrassment from mistakes drives rapid correction, as observed in training a puppy to avoid repeating errors after a single scolding CosmoCore tags code generation trajectories with valence and surprise using a lightweight multi-layer perceptron (MLP). High-negative valence (cringe) episodes, such as buggy code outputs, are prioritized in a Dream Queue for five-fold replay during off-policy updates, while low-surprise successes are pruned to prevent overconfidence and buffer bloat. Evaluated on code generation benchmarks like HumanEval and BigCodeBench, alongside simulations with a custom data pipeline environment, CosmoCore reduces hallucinated code (e.g., syntax errors or logical bugs) by 48\\% and accelerates self-correction by 45\\%. Local experiments using Hugging Face models in a PySpark environment validate these gains, with code snippets provided for replication. Ablations confirm valence tagging boosts curiosity in exploration, and pruning mitigates inefficiency. This framework extends RL from human feedback (RLHF) for more emotionally aware code assistants, with applications in IDEs and data pipelines. Code and the custom mini-world simulation are released.",
      "arxiv_id": "2510.18895v1",
      "published": "2025-10-20",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2510.18895v1",
      "pdf_url": "https://arxiv.org/pdf/2510.18895v1.pdf"
    },
    {
      "title": "Distributional Preference Learning: Understanding and Accounting for Hidden Context in RLHF",
      "authors": [
        "Anand Siththaranjan",
        "Cassidy Laidlaw",
        "Dylan Hadfield-Menell"
      ],
      "summary": "In practice, preference learning from human feedback depends on incomplete data with hidden context. Hidden context refers to data that affects the feedback received, but which is not represented in the data used to train a preference model. This captures common issues of data collection, such as having human annotators with varied preferences, cognitive processes that result in seemingly irrational behavior, and combining data labeled according to different criteria. We prove that standard applications of preference learning, including reinforcement learning from human feedback (RLHF), implicitly aggregate over hidden contexts according to a well-known voting rule called Borda count. We show this can produce counter-intuitive results that are very different from other methods which implicitly aggregate via expected utility. Furthermore, our analysis formalizes the way that preference learning from users with diverse values tacitly implements a social choice function. A key implication of this result is that annotators have an incentive to misreport their preferences in order to influence the learned model, leading to vulnerabilities in the deployment of RLHF. As a step towards mitigating these problems, we introduce a class of methods called distributional preference learning (DPL). DPL methods estimate a distribution of possible score values for each alternative in order to better account for hidden context. Experimental results indicate that applying DPL to RLHF for LLM chatbots identifies hidden context in the data and significantly reduces subsequent jailbreak vulnerability. Our code and data are available at https://github.com/cassidylaidlaw/hidden-context",
      "arxiv_id": "2312.08358v2",
      "published": "2023-12-13",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2312.08358v2",
      "pdf_url": "https://arxiv.org/pdf/2312.08358v2.pdf"
    },
    {
      "title": "Learning a Pessimistic Reward Model in RLHF",
      "authors": [
        "Yinglun Xu",
        "Hangoo Kang",
        "Tarun Suresh",
        "Yuxuan Wan",
        "Gagandeep Singh"
      ],
      "summary": "This work proposes `PET', a novel pessimistic reward fine-tuning method, to learn a pessimistic reward model robust against reward hacking in offline reinforcement learning from human feedback (RLHF). Traditional reward modeling techniques in RLHF train an imperfect reward model, on which a KL regularization plays a pivotal role in mitigating reward hacking when optimizing a policy. Such an intuition-based method still suffers from reward hacking, and the policies with large KL divergence from the dataset distribution are excluded during learning. In contrast, we show that when optimizing a policy on a pessimistic reward model fine-tuned through PET, reward hacking can be prevented without relying on any regularization. We test our methods on the standard TL;DR summarization dataset. We find that one can learn a high-quality policy on our pessimistic reward without using any regularization. Such a policy has a high KL divergence from the dataset distribution while having high performance in practice. In summary, our work shows the feasibility of learning a pessimistic reward model against reward hacking. The agent can greedily search for the policy with a high pessimistic reward without suffering from reward hacking.",
      "arxiv_id": "2505.20556v1",
      "published": "2025-05-26",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.20556v1",
      "pdf_url": "https://arxiv.org/pdf/2505.20556v1.pdf"
    },
    {
      "title": "Can Differentiable Decision Trees Enable Interpretable Reward Learning from Human Feedback?",
      "authors": [
        "Akansha Kalra",
        "Daniel S. Brown"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a popular paradigm for capturing human intent to alleviate the challenges of hand-crafting the reward values. Despite the increasing interest in RLHF, most works learn black box reward functions that while expressive are difficult to interpret and often require running the whole costly process of RL before we can even decipher if these frameworks are actually aligned with human preferences. We propose and evaluate a novel approach for learning expressive and interpretable reward functions from preferences using Differentiable Decision Trees (DDTs). Our experiments across several domains, including CartPole, Visual Gridworld environments and Atari games, provide evidence that the tree structure of our learned reward function is useful in determining the extent to which the reward function is aligned with human preferences. We also provide experimental evidence that not only shows that reward DDTs can often achieve competitive RL performance when compared with larger capacity deep neural network reward functions but also demonstrates the diagnostic utility of our framework in checking alignment of learned reward functions. We also observe that the choice between soft and hard (argmax) output of reward DDT reveals a tension between wanting highly shaped rewards to ensure good RL performance, while also wanting simpler, more interpretable rewards. Videos and code, are available at: https://sites.google.com/view/ddt-rlhf",
      "arxiv_id": "2306.13004v6",
      "published": "2023-06-22",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2306.13004v6",
      "pdf_url": "https://arxiv.org/pdf/2306.13004v6.pdf"
    },
    {
      "title": "Understanding the Performance Gap in Preference Learning: A Dichotomy of RLHF and DPO",
      "authors": [
        "Ruizhe Shi",
        "Minhak Song",
        "Runlong Zhou",
        "Zihan Zhang",
        "Maryam Fazel",
        "Simon S. Du"
      ],
      "summary": "We present a fine-grained theoretical analysis of the performance gap between reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO) under a representation gap. Our study decomposes this gap into two sources: an explicit representation gap under exact optimization and an implicit representation gap under finite samples. In the exact optimization setting, we characterize how the relative capacities of the reward and policy model classes influence the final policy qualities. We show that RLHF, DPO, or online DPO can outperform one another depending on type of model mis-specifications. Notably, online DPO can outperform both RLHF and standard DPO when the reward and policy model classes are isomorphic and both mis-specified. In the approximate optimization setting, we provide a concrete construction where the ground-truth reward is implicitly sparse and show that RLHF requires significantly fewer samples than DPO to recover an effective reward model -- highlighting a statistical advantage of two-stage learning. Together, these results provide a comprehensive understanding of the performance gap between RLHF and DPO under various settings, and offer practical insights into when each method is preferred.",
      "arxiv_id": "2505.19770v2",
      "published": "2025-05-26",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.19770v2",
      "pdf_url": "https://arxiv.org/pdf/2505.19770v2.pdf"
    },
    {
      "title": "Klear-CodeTest: Scalable Test Case Generation for Code Reinforcement Learning",
      "authors": [
        "Jia Fu",
        "Xinyu Yang",
        "Hongzhi Zhang",
        "Yahui Liu",
        "Jingyuan Zhang",
        "Qi Wang",
        "Fuzheng Zhang",
        "Guorui Zhou"
      ],
      "summary": "Precise, correct feedback is crucial for effectively training large language models (LLMs) in code reinforcement learning. However, synthesizing high-quality test cases remains a profoundly challenging and unsolved problem. In this work, we present Klear-CodeTest, a comprehensive test case synthesis framework featuring rigorous verification to ensure quality and reliability of test cases. Our approach achieves broad coverage of programming problems via a novel Generator-Validation (G-V) framework, ensuring correctness through a consistency validation mechanism that verifies outputs against gold solutions. The proposed G-V framework generates comprehensive test cases including both regular and corner cases, enhancing test coverage and discriminative power for solution correctness assessment in code reinforcement learning. In addition, we design a multi-layered security sandbox system optimized for online verification platforms, guaranteeing safe and reliable code execution. Through comprehensive experiments, we demonstrate the effectiveness of our curated dataset, showing significant improvements in model performance and training stability. The source codes, curated dataset and sandbox system are available at: https://github.com/Kwai-Klear/CodeTest.",
      "arxiv_id": "2508.05710v2",
      "published": "2025-08-07",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.05710v2",
      "pdf_url": "https://arxiv.org/pdf/2508.05710v2.pdf"
    },
    {
      "title": "Generative RLHF-V: Learning Principles from Multi-modal Human Preference",
      "authors": [
        "Jiayi Zhou",
        "Jiaming Ji",
        "Boyuan Chen",
        "Jiapeng Sun",
        "Wenqi Chen",
        "Donghai Hong",
        "Sirui Han",
        "Yike Guo",
        "Yaodong Yang"
      ],
      "summary": "Training multi-modal large language models (MLLMs) that align with human intentions is a long-term challenge. Traditional score-only reward models for alignment suffer from low accuracy, weak generalization, and poor interpretability, blocking the progress of alignment methods, e.g., reinforcement learning from human feedback (RLHF). Generative reward models (GRMs) leverage MLLMs' intrinsic reasoning capabilities to discriminate pair-wise responses, but their pair-wise paradigm makes it hard to generalize to learnable rewards. We introduce Generative RLHF-V, a novel alignment framework that integrates GRMs with multi-modal RLHF. We propose a two-stage pipeline: $\\textbf{multi-modal generative reward modeling from RL}$, where RL guides GRMs to actively capture human intention, then predict the correct pair-wise scores; and $\\textbf{RL optimization from grouped comparison}$, which enhances multi-modal RL scoring precision by grouped responses comparison. Experimental results demonstrate that, besides out-of-distribution generalization of RM discrimination, our framework improves 4 MLLMs' performance across 7 benchmarks by $18.1\\%$, while the baseline RLHF is only $5.3\\%$. We further validate that Generative RLHF-V achieves a near-linear improvement with an increasing number of candidate responses. Our code and models can be found at https://generative-rlhf-v.github.io.",
      "arxiv_id": "2505.18531v1",
      "published": "2025-05-24",
      "categories": [
        "cs.AI",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2505.18531v1",
      "pdf_url": "https://arxiv.org/pdf/2505.18531v1.pdf"
    },
    {
      "title": "GALOIS: Boosting Deep Reinforcement Learning via Generalizable Logic Synthesis",
      "authors": [
        "Yushi Cao",
        "Zhiming Li",
        "Tianpei Yang",
        "Hao Zhang",
        "Yan Zheng",
        "Yi Li",
        "Jianye Hao",
        "Yang Liu"
      ],
      "summary": "Despite achieving superior performance in human-level control problems, unlike humans, deep reinforcement learning (DRL) lacks high-order intelligence (e.g., logic deduction and reuse), thus it behaves ineffectively than humans regarding learning and generalization in complex problems. Previous works attempt to directly synthesize a white-box logic program as the DRL policy, manifesting logic-driven behaviors. However, most synthesis methods are built on imperative or declarative programming, and each has a distinct limitation, respectively. The former ignores the cause-effect logic during synthesis, resulting in low generalizability across tasks. The latter is strictly proof-based, thus failing to synthesize programs with complex hierarchical logic. In this paper, we combine the above two paradigms together and propose a novel Generalizable Logic Synthesis (GALOIS) framework to synthesize hierarchical and strict cause-effect logic programs. GALOIS leverages the program sketch and defines a new sketch-based hybrid program language for guiding the synthesis. Based on that, GALOIS proposes a sketch-based program synthesis method to automatically generate white-box programs with generalizable and interpretable cause-effect logic. Extensive evaluations on various decision-making tasks with complex logic demonstrate the superiority of GALOIS over mainstream baselines regarding the asymptotic performance, generalizability, and great knowledge reusability across different environments.",
      "arxiv_id": "2205.13728v1",
      "published": "2022-05-27",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2205.13728v1",
      "pdf_url": "https://arxiv.org/pdf/2205.13728v1.pdf"
    },
    {
      "title": "Measuring memorization in RLHF for code completion",
      "authors": [
        "Aneesh Pappu",
        "Billy Porter",
        "Ilia Shumailov",
        "Jamie Hayes"
      ],
      "summary": "Reinforcement learning with human feedback (RLHF) has become the dominant method to align large models to user preferences. Unlike fine-tuning, for which there are many studies regarding training data memorization, it is not clear how memorization is affected by or introduced in the RLHF alignment process. Understanding this relationship is important as real user data may be collected and used to align large models; if user data is memorized during RLHF and later regurgitated, this could raise privacy concerns. In addition to RLHF, other methods such as Direct Preference Optimization (DPO) and $\u03a8$PO have gained popularity for learning directly from human preferences, removing the need for optimizing intermediary reward models with reinforcement learning. In this work, we analyze how training data memorization can surface and propagate through each phase of RLHF and direct preference learning. We focus our study on code completion models, as code completion is one of the most popular use cases for large language models. We find that RLHF significantly decreases the chance that data used for reward modeling and reinforcement learning is memorized in comparison to directly fine-tuning on this data, but that examples already memorized during the fine-tuning stage of RLHF, will, in the majority of cases, remain memorized after RLHF. In contrast, we find that aligning by learning directly from human preference data via a special case of $\u03a8$PO, Identity Preference Optimization (IPO), increases the likelihood that training data is regurgitated compared to RLHF. Our work suggests that RLHF, as opposed to direct preference learning, is a safer way to mitigate the risk of regurgitating sensitive preference data when aligning large language models. We find our conclusions are robust across multiple code completion datasets, tasks, and model scales.",
      "arxiv_id": "2406.11715v2",
      "published": "2024-06-17",
      "categories": [
        "cs.LG",
        "cs.CL",
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2406.11715v2",
      "pdf_url": "https://arxiv.org/pdf/2406.11715v2.pdf"
    },
    {
      "title": "ReCode: Updating Code API Knowledge with Reinforcement Learning",
      "authors": [
        "Haoze Wu",
        "Yunzhi Yao",
        "Wenhao Yu",
        "Ningyu Zhang"
      ],
      "summary": "Large Language Models (LLMs) exhibit remarkable code generation capabilities but falter when adapting to frequent updates in external library APIs. This critical limitation, stemming from reliance on outdated API knowledge from their training data, even with access to current documentation, impedes reliable code generation in dynamic environments. To tackle this issue, we propose ReCode (rule-based Reinforcement learning for Code Update), a novel framework that mimics human programmer adaptation to API changes. Specifically, we construct a dataset of approximately 2,000 data entries to train the LLMs to perform version migration based on updated information. Then, we introduce a modified string similarity metric for code evaluation as the reward for reinforcement learning. Our experiments demonstrate that ReCode substantially boosts LLMs' code generation performance in dynamic API scenarios, especially on the unseen CodeUpdateArena task. Crucially, compared to supervised fine-tuning, ReCode has less impact on LLMs' general code generation abilities. We apply ReCode on various LLMs and reinforcement learning algorithms (GRPO and DAPO), all achieving consistent improvements. Notably, after training, Qwen2.5-Coder-7B outperforms that of the 32B parameter code instruction-tuned model and the reasoning model with the same architecture. Code is available at https://github.com/zjunlp/ReCode.",
      "arxiv_id": "2506.20495v4",
      "published": "2025-06-25",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.IR",
        "cs.LG",
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2506.20495v4",
      "pdf_url": "https://arxiv.org/pdf/2506.20495v4.pdf"
    },
    {
      "title": "Improved Tree Search for Automatic Program Synthesis",
      "authors": [
        "Aran Carmon",
        "Lior Wolf"
      ],
      "summary": "In the task of automatic program synthesis, one obtains pairs of matching inputs and outputs and generates a computer program, in a particular domain-specific language (DSL), which given each sample input returns the matching output. A key element is being able to perform an efficient search in the space of valid programs. Here, we suggest a variant of MCTS that leads to state of the art results on two vastly different DSLs. The exploration method we propose includes multiple contributions: a modified visit count, a preprocessing procedure for the training dataset, and encoding the part of the program that was already executed.",
      "arxiv_id": "2303.07166v1",
      "published": "2023-03-13",
      "categories": [
        "cs.LG",
        "cs.PL",
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2303.07166v1",
      "pdf_url": "https://arxiv.org/pdf/2303.07166v1.pdf"
    },
    {
      "title": "Enhancing Code LLMs with Reinforcement Learning in Code Generation: A Survey",
      "authors": [
        "Junqiao Wang",
        "Zeng Zhang",
        "Yangfan He",
        "Zihao Zhang",
        "Xinyuan Song",
        "Yuyang Song",
        "Tianyu Shi",
        "Yuchen Li",
        "Hengyuan Xu",
        "Kunyu Wu",
        "Xin Yi",
        "Zhongwei Wan",
        "Xinhang Yuan",
        "Zijun Wang",
        "Kuan Lu",
        "Menghao Huo",
        "Tang Jingqun",
        "Guangwu Qian",
        "Keqin Li",
        "Qiuwu Chen",
        "Lewei He"
      ],
      "summary": "With the rapid evolution of large language models (LLM), reinforcement learning (RL) has emerged as a pivotal technique for code generation and optimization in various domains. This paper presents a systematic survey of the application of RL in code optimization and generation, highlighting its role in enhancing compiler optimization, resource allocation, and the development of frameworks and tools. Subsequent sections first delve into the intricate processes of compiler optimization, where RL algorithms are leveraged to improve efficiency and resource utilization. The discussion then progresses to the function of RL in resource allocation, emphasizing register allocation and system optimization. We also explore the burgeoning role of frameworks and tools in code generation, examining how RL can be integrated to bolster their capabilities. This survey aims to serve as a comprehensive resource for researchers and practitioners interested in harnessing the power of RL to advance code generation and optimization techniques.",
      "arxiv_id": "2412.20367v5",
      "published": "2024-12-29",
      "categories": [
        "cs.SE",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2412.20367v5",
      "pdf_url": "https://arxiv.org/pdf/2412.20367v5.pdf"
    },
    {
      "title": "Proof2Silicon: Prompt Repair for Verified Code and Hardware Generation via Reinforcement Learning",
      "authors": [
        "Manvi Jha",
        "Jiaxin Wan",
        "Deming Chen"
      ],
      "summary": "Large Language Models (LLMs) have demonstrated impressive capabilities in automated code generation but frequently produce code that fails formal verification, an essential requirement for hardware and safety-critical domains. To overcome this fundamental limitation, we previously proposed PREFACE, a model-agnostic framework based on reinforcement learning (RL) that iteratively repairs the prompts provided to frozen LLMs, systematically steering them toward generating formally verifiable Dafny code without costly fine-tuning. This work presents Proof2Silicon, a novel end-to-end synthesis framework that embeds the previously proposed PREFACE flow to enable the generation of correctness-by-construction hardware directly from natural language specifications. Proof2Silicon operates by: (1) leveraging PREFACE's verifier-driven RL agent to optimize prompt generation iteratively, ensuring Dafny code correctness; (2) automatically translating verified Dafny programs into synthesizable high-level C using Dafny's Python backend and PyLog; and (3) employing Vivado HLS to produce RTL implementations. Evaluated rigorously on a challenging 100-task benchmark, PREFACE's RL-guided prompt optimization consistently improved Dafny verification success rates across diverse LLMs by up to 21%. Crucially, Proof2Silicon achieved an end-to-end hardware synthesis success rate of up to 72%, generating RTL designs through Vivado HLS synthesis flows. These results demonstrate a robust, scalable, and automated pipeline for LLM-driven, formally verified hardware synthesis, bridging natural-language specification and silicon realization.",
      "arxiv_id": "2509.06239v2",
      "published": "2025-09-07",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2509.06239v2",
      "pdf_url": "https://arxiv.org/pdf/2509.06239v2.pdf"
    },
    {
      "title": "SharedRep-RLHF: A Shared Representation Approach to RLHF with Diverse Preferences",
      "authors": [
        "Arpan Mukherjee",
        "Marcello Bullo",
        "Deniz G\u00fcnd\u00fcz"
      ],
      "summary": "Uniform-reward reinforcement learning from human feedback (RLHF), which trains a single reward model to represent the preferences of all annotators, fails to capture the diversity of opinions across sub-populations, inadvertently favoring dominant groups. The state-of-the-art, MaxMin-RLHF, addresses this by learning group-specific reward models, and by optimizing for the group receiving the minimum reward, thereby promoting fairness. However, we identify that a key limitation of MaxMin-RLHF is its poor performance when the minimum-reward group is a minority. To mitigate this drawback, we introduce a novel framework, termed {\\em SharedRep-RLHF}. At its core, SharedRep-RLHF learns and leverages {\\em shared traits} in annotations among various groups, in contrast to learning separate reward models across groups. We first show that MaxMin-RLHF is provably suboptimal in learning shared traits, and then quantify the sample complexity of SharedRep-RLHF. Experiments across diverse natural language tasks showcase the effectiveness of SharedRep-RLHF compared to MaxMin-RLHF with a gain of up to 20% in win rate.",
      "arxiv_id": "2509.03672v1",
      "published": "2025-09-03",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2509.03672v1",
      "pdf_url": "https://arxiv.org/pdf/2509.03672v1.pdf"
    },
    {
      "title": "Gradual Transition from Bellman Optimality Operator to Bellman Operator in Online Reinforcement Learning",
      "authors": [
        "Motoki Omura",
        "Kazuki Ota",
        "Takayuki Osa",
        "Yusuke Mukuta",
        "Tatsuya Harada"
      ],
      "summary": "For continuous action spaces, actor-critic methods are widely used in online reinforcement learning (RL). However, unlike RL algorithms for discrete actions, which generally model the optimal value function using the Bellman optimality operator, RL algorithms for continuous actions typically model Q-values for the current policy using the Bellman operator. These algorithms for continuous actions rely exclusively on policy updates for improvement, which often results in low sample efficiency. This study examines the effectiveness of incorporating the Bellman optimality operator into actor-critic frameworks. Experiments in a simple environment show that modeling optimal values accelerates learning but leads to overestimation bias. To address this, we propose an annealing approach that gradually transitions from the Bellman optimality operator to the Bellman operator, thereby accelerating learning while mitigating bias. Our method, combined with TD3 and SAC, significantly outperforms existing approaches across various locomotion and manipulation tasks, demonstrating improved performance and robustness to hyperparameters related to optimality. The code for this study is available at https://github.com/motokiomura/annealed-q-learning.",
      "arxiv_id": "2506.05968v2",
      "published": "2025-06-06",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2506.05968v2",
      "pdf_url": "https://arxiv.org/pdf/2506.05968v2.pdf"
    },
    {
      "title": "Large Language Model for Verilog Generation with Code-Structure-Guided Reinforcement Learning",
      "authors": [
        "Ning Wang",
        "Bingkun Yao",
        "Jie Zhou",
        "Xi Wang",
        "Zhe Jiang",
        "Nan Guan"
      ],
      "summary": "Recent advancements in large language models (LLMs) have sparked significant interest in the automatic generation of Register Transfer Level (RTL) designs, particularly using Verilog. Current research on this topic primarily focuses on pre-training and instruction tuning, but the effectiveness of these methods is constrained by the limited availability of training data, as public Verilog code is far less abundant than software code. In particular, these methods struggle to effectively capture Verilog parallel code structures, which fundamentally differ from the imperative, sequential control flow typical in most software programming languages. This paper introduces VeriSeek, an LLM enhanced by reinforcement learning using a limited amount of high-quality training data to achieve high Verilog code generation performance. Our reinforcement learning approach employs code structure information as feedback signals to refine the pre-trained model, enabling it to effectively learn important patterns from Verilog code with parallel structures. Experiments show that VeriSeek outperforms state-of-the-art methods across multiple benchmarks.",
      "arxiv_id": "2407.18271v4",
      "published": "2024-07-21",
      "categories": [
        "cs.AR",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2407.18271v4",
      "pdf_url": "https://arxiv.org/pdf/2407.18271v4.pdf"
    },
    {
      "title": "HybridFlow: A Flexible and Efficient RLHF Framework",
      "authors": [
        "Guangming Sheng",
        "Chi Zhang",
        "Zilingfeng Ye",
        "Xibin Wu",
        "Wang Zhang",
        "Ru Zhang",
        "Yanghua Peng",
        "Haibin Lin",
        "Chuan Wu"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is widely used in Large Language Model (LLM) alignment. Traditional RL can be modeled as a dataflow, where each node represents computation of a neural network (NN) and each edge denotes data dependencies between the NNs. RLHF complicates the dataflow by expanding each node into a distributed LLM training or generation program, and each edge into a many-to-many multicast. Traditional RL frameworks execute the dataflow using a single controller to instruct both intra-node computation and inter-node communication, which can be inefficient in RLHF due to large control dispatch overhead for distributed intra-node computation. Existing RLHF systems adopt a multi-controller paradigm, which can be inflexible due to nesting distributed computation and data communication. We propose HybridFlow, which combines single-controller and multi-controller paradigms in a hybrid manner to enable flexible representation and efficient execution of the RLHF dataflow. We carefully design a set of hierarchical APIs that decouple and encapsulate computation and data dependencies in the complex RLHF dataflow, allowing efficient operation orchestration to implement RLHF algorithms and flexible mapping of the computation onto various devices. We further design a 3D-HybridEngine for efficient actor model resharding between training and generation phases, with zero memory redundancy and significantly reduced communication overhead. Our experimental results demonstrate 1.53$\\times$~20.57$\\times$ throughput improvement when running various RLHF algorithms using HybridFlow, as compared with state-of-the-art baselines. HybridFlow source code will be available at https://github.com/volcengine/verl.",
      "arxiv_id": "2409.19256v2",
      "published": "2024-09-28",
      "categories": [
        "cs.LG",
        "cs.DC"
      ],
      "url": "https://arxiv.org/abs/2409.19256v2",
      "pdf_url": "https://arxiv.org/pdf/2409.19256v2.pdf"
    },
    {
      "title": "From Intent to Execution: Multimodal Chain-of-Thought Reinforcement Learning for Precise CAD Code Generation",
      "authors": [
        "Ke Niu",
        "Haiyang Yu",
        "Zhuofan Chen",
        "Mengyang Zhao",
        "Teng Fu",
        "Bin Li",
        "Xiangyang Xue"
      ],
      "summary": "Computer-Aided Design (CAD) plays a vital role in engineering and manufacturing, yet current CAD workflows require extensive domain expertise and manual modeling effort. Recent advances in large language models (LLMs) have made it possible to generate code from natural language, opening new opportunities for automating parametric 3D modeling. However, directly translating human design intent into executable CAD code remains highly challenging, due to the need for logical reasoning, syntactic correctness, and numerical precision. In this work, we propose CAD-RL, a multimodal Chain-of-Thought (CoT) guided reinforcement learning post training framework for CAD modeling code generation. Our method combines CoT-based Cold Start with goal-driven reinforcement learning post training using three task-specific rewards: executability reward, geometric accuracy reward, and external evaluation reward. To ensure stable policy learning under sparse and high-variance reward conditions, we introduce three targeted optimization strategies: Trust Region Stretch for improved exploration, Precision Token Loss for enhanced dimensions parameter accuracy, and Overlong Filtering to reduce noisy supervision. To support training and benchmarking, we release ExeCAD, a noval dataset comprising 16,540 real-world CAD examples with paired natural language and structured design language descriptions, executable CADQuery scripts, and rendered 3D models. Experiments demonstrate that CAD-RL achieves significant improvements in reasoning quality, output precision, and code executability over existing VLMs.",
      "arxiv_id": "2508.10118v2",
      "published": "2025-08-13",
      "categories": [
        "cs.LG",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2508.10118v2",
      "pdf_url": "https://arxiv.org/pdf/2508.10118v2.pdf"
    },
    {
      "title": "Improving Environment Robustness of Deep Reinforcement Learning Approaches for Autonomous Racing Using Bayesian Optimization-based Curriculum Learning",
      "authors": [
        "Rohan Banerjee",
        "Prishita Ray",
        "Mark Campbell"
      ],
      "summary": "Deep reinforcement learning (RL) approaches have been broadly applied to a large number of robotics tasks, such as robot manipulation and autonomous driving. However, an open problem in deep RL is learning policies that are robust to variations in the environment, which is an important condition for such systems to be deployed into real-world, unstructured settings. Curriculum learning is one approach that has been applied to improve generalization performance in both supervised and reinforcement learning domains, but selecting the appropriate curriculum to achieve robustness can be a user-intensive process. In our work, we show that performing probabilistic inference of the underlying curriculum-reward function using Bayesian Optimization can be a promising technique for finding a robust curriculum. We demonstrate that a curriculum found with Bayesian optimization can outperform a vanilla deep RL agent and a hand-engineered curriculum in the domain of autonomous racing with obstacle avoidance. Our code is available at https://github.com/PRISHIta123/Curriculum_RL_for_Driving.",
      "arxiv_id": "2312.10557v1",
      "published": "2023-12-16",
      "categories": [
        "cs.RO",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2312.10557v1",
      "pdf_url": "https://arxiv.org/pdf/2312.10557v1.pdf"
    },
    {
      "title": "Learning Game-Playing Agents with Generative Code Optimization",
      "authors": [
        "Zhiyi Kuang",
        "Ryan Rong",
        "YuCheng Yuan",
        "Allen Nie"
      ],
      "summary": "We present a generative optimization approach for learning game-playing agents, where policies are represented as Python programs and refined using large language models (LLMs). Our method treats decision-making policies as self-evolving code, with current observation as input and an in-game action as output, enabling agents to self-improve through execution traces and natural language feedback with minimal human intervention. Applied to Atari games, our game-playing Python program achieves performance competitive with deep reinforcement learning (RL) baselines while using significantly less training time and much fewer environment interactions. This work highlights the promise of programmatic policy representations for building efficient, adaptable agents capable of complex, long-horizon reasoning.",
      "arxiv_id": "2508.19506v1",
      "published": "2025-08-27",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.19506v1",
      "pdf_url": "https://arxiv.org/pdf/2508.19506v1.pdf"
    },
    {
      "title": "Aligning Crowd-sourced Human Feedback for Reinforcement Learning on Code Generation by Large Language Models",
      "authors": [
        "Man Fai Wong",
        "Chee Wei Tan"
      ],
      "summary": "This paper studies how AI-assisted programming and large language models (LLM) improve software developers' ability via AI tools (LLM agents) like Github Copilot and Amazon CodeWhisperer, while integrating human feedback to enhance reinforcement learning (RLHF) with crowd-sourced computation to enhance text-to-code generation. Additionally, we demonstrate that our Bayesian optimization framework supports AI alignment in code generation by distributing the feedback collection burden, highlighting the value of collecting human feedback of good quality. Our empirical evaluations demonstrate the efficacy of this approach, showcasing how LLM agents can be effectively trained for improved text-to-code generation. Our Bayesian optimization framework can be designed for general domain-specific languages, promoting the alignment of large language model capabilities with human feedback in AI-assisted programming for code generation.",
      "arxiv_id": "2503.15129v1",
      "published": "2025-03-19",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.15129v1",
      "pdf_url": "https://arxiv.org/pdf/2503.15129v1.pdf"
    },
    {
      "title": "Sharp Analysis for KL-Regularized Contextual Bandits and RLHF",
      "authors": [
        "Heyang Zhao",
        "Chenlu Ye",
        "Quanquan Gu",
        "Tong Zhang"
      ],
      "summary": "Reverse-Kullback-Leibler (KL) regularization has emerged to be a predominant technique used to enhance policy optimization in reinforcement learning (RL) and reinforcement learning from human feedback (RLHF), which forces the learned policy to stay close to a reference policy. While the effectiveness and necessity of KL-regularization have been empirically demonstrated in various practical scenarios, current theoretical analysis of KL-regularized RLHF still obtains the same $\\mathcal{O}(1 / \u03b5^2)$ sample complexity as problems without KL-regularization. To understand the fundamental distinction between policy learning objectives with KL-regularization and ones without KL-regularization, we are the first to theoretically demonstrate the power of KL-regularization by providing a sharp analysis for KL-regularized contextual bandits and RLHF, revealing an $\\mathcal{O}(1 / \u03b5)$ sample complexity when $\u03b5$ is sufficiently small.   We further explore the role of data coverage in contextual bandits and RLHF. While the coverage assumption is commonly employed in offline RLHF to link the samples from the reference policy to the optimal policy, often at the cost of a multiplicative dependence on the coverage coefficient, its impact on the sample complexity of online RLHF remains unclear. Previous theoretical analyses of online RLHF typically require explicit exploration and additional structural assumptions on the reward function class. In contrast, we show that with sufficient coverage from the reference policy, a simple two-stage mixed sampling strategy can achieve a sample complexity with only an additive dependence on the coverage coefficient. Our results provide a comprehensive understanding of the roles of KL-regularization and data coverage in RLHF, shedding light on the design of more efficient RLHF algorithms.",
      "arxiv_id": "2411.04625v2",
      "published": "2024-11-07",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2411.04625v2",
      "pdf_url": "https://arxiv.org/pdf/2411.04625v2.pdf"
    },
    {
      "title": "Coarse-Tuning Models of Code with Reinforcement Learning Feedback",
      "authors": [
        "Abhinav Jain",
        "Chima Adiole",
        "Swarat Chaudhuri",
        "Thomas Reps",
        "Chris Jermaine"
      ],
      "summary": "Large Language Models (LLMs) pre-trained on code have recently emerged as the dominant approach to program synthesis. However, these models are trained using next-token prediction, which ignores the syntax and semantics of code. We propose RLCF, that further trains a pre-trained LLM via reinforcement learning, using feedback from a grounding function that scores the quality of the code. The grounding function uses (i) compiler-derived feedback on whether the code it generates passes a set of correctness checks; and (ii) feedback from a different LLM that compares the generated code to a reference code. RLCF is model- and language-agnostic. We empirically evaluate it on the MBJP and MathQA tasks for Java. Our experiments show that RLCF raises the odds that an LLM-generated program compiles, is executable, and produces the right output on tests, often allowing LLMs to match the performance of 2x-8x larger LLMs.",
      "arxiv_id": "2305.18341v2",
      "published": "2023-05-25",
      "categories": [
        "cs.PL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2305.18341v2",
      "pdf_url": "https://arxiv.org/pdf/2305.18341v2.pdf"
    },
    {
      "title": "MathDSL: A Domain-Specific Language for Concise Mathematical Solutions Via Program Synthesis",
      "authors": [
        "Sagnik Anupam",
        "Maddy Bowers",
        "Omar Costilla-Reyes",
        "Armando Solar-Lezama"
      ],
      "summary": "We present MathDSL, a Domain-Specific Language (DSL) for mathematical equation solving, which, when deployed in program synthesis models, outperforms state-of-the-art reinforcement-learning-based methods. We also introduce a quantitative metric for measuring the conciseness of a mathematical solution and demonstrate the improvement in the quality of generated solutions compared to other methods. Our system demonstrates that a program synthesis system (DreamCoder) using MathDSL can generate programs that solve linear equations with greater accuracy and conciseness than using reinforcement learning systems. Additionally, we demonstrate that if we use the action spaces of previous reinforcement learning systems as DSLs, MathDSL outperforms the action-space-DSLs. We use DreamCoder to store equation-solving strategies as learned abstractions in its program library and demonstrate that by using MathDSL, these can be converted into human-interpretable solution strategies that could have applications in mathematical education.",
      "arxiv_id": "2409.17490v3",
      "published": "2024-09-26",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2409.17490v3",
      "pdf_url": "https://arxiv.org/pdf/2409.17490v3.pdf"
    },
    {
      "title": "Reinforcement Learning from Human Feedback: Whose Culture, Whose Values, Whose Perspectives?",
      "authors": [
        "Kristian Gonz\u00e1lez Barman",
        "Simon Lohse",
        "Henk de Regt"
      ],
      "summary": "We argue for the epistemic and ethical advantages of pluralism in Reinforcement Learning from Human Feedback (RLHF) in the context of Large Language Models (LLM). Drawing on social epistemology and pluralist philosophy of science, we suggest ways in which RHLF can be made more responsive to human needs and how we can address challenges along the way. The paper concludes with an agenda for change, i.e. concrete, actionable steps to improve LLM development.",
      "arxiv_id": "2407.17482v2",
      "published": "2024-07-02",
      "categories": [
        "cs.CY",
        "cs.AI",
        "cs.CL",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2407.17482v2",
      "pdf_url": "https://arxiv.org/pdf/2407.17482v2.pdf"
    },
    {
      "title": "StepCoder: Improve Code Generation with Reinforcement Learning from Compiler Feedback",
      "authors": [
        "Shihan Dou",
        "Yan Liu",
        "Haoxiang Jia",
        "Limao Xiong",
        "Enyu Zhou",
        "Wei Shen",
        "Junjie Shan",
        "Caishuang Huang",
        "Xiao Wang",
        "Xiaoran Fan",
        "Zhiheng Xi",
        "Yuhao Zhou",
        "Tao Ji",
        "Rui Zheng",
        "Qi Zhang",
        "Xuanjing Huang",
        "Tao Gui"
      ],
      "summary": "The advancement of large language models (LLMs) has significantly propelled the field of code generation. Previous work integrated reinforcement learning (RL) with compiler feedback for exploring the output space of LLMs to enhance code generation quality. However, the lengthy code generated by LLMs in response to complex human requirements makes RL exploration a challenge. Also, since the unit tests may not cover the complicated code, optimizing LLMs by using these unexecuted code snippets is ineffective. To tackle these challenges, we introduce StepCoder, a novel RL framework for code generation, consisting of two main components: CCCS addresses the exploration challenge by breaking the long sequences code generation task into a Curriculum of Code Completion Subtasks, while FGO only optimizes the model by masking the unexecuted code segments to provide Fine-Grained Optimization. In addition, we furthermore construct the APPS+ dataset for RL training, which is manually verified to ensure the correctness of unit tests. Experimental results show that our method improves the ability to explore the output space and outperforms state-of-the-art approaches in corresponding benchmarks. Our dataset APPS+ and StepCoder are available online.",
      "arxiv_id": "2402.01391v2",
      "published": "2024-02-02",
      "categories": [
        "cs.SE",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2402.01391v2",
      "pdf_url": "https://arxiv.org/pdf/2402.01391v2.pdf"
    },
    {
      "title": "LLM Misalignment via Adversarial RLHF Platforms",
      "authors": [
        "Erfan Entezami",
        "Ali Naseh"
      ],
      "summary": "Reinforcement learning has shown remarkable performance in aligning language models with human preferences, leading to the rise of attention towards developing RLHF platforms. These platforms enable users to fine-tune models without requiring any expertise in developing complex machine learning algorithms. While these platforms offer useful features such as reward modeling and RLHF fine-tuning, their security and reliability remain largely unexplored. Given the growing adoption of RLHF and open-source RLHF frameworks, we investigate the trustworthiness of these systems and their potential impact on behavior of LLMs. In this paper, we present an attack targeting publicly available RLHF tools. In our proposed attack, an adversarial RLHF platform corrupts the LLM alignment process by selectively manipulating data samples in the preference dataset. In this scenario, when a user's task aligns with the attacker's objective, the platform manipulates a subset of the preference dataset that contains samples related to the attacker's target. This manipulation results in a corrupted reward model, which ultimately leads to the misalignment of the language model. Our results demonstrate that such an attack can effectively steer LLMs toward undesirable behaviors within the targeted domains. Our work highlights the critical need to explore the vulnerabilities of RLHF platforms and their potential to cause misalignment in LLMs during the RLHF fine-tuning process.",
      "arxiv_id": "2503.03039v1",
      "published": "2025-03-04",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.03039v1",
      "pdf_url": "https://arxiv.org/pdf/2503.03039v1.pdf"
    },
    {
      "title": "VERIRL: Boosting the LLM-based Verilog Code Generation via Reinforcement Learning",
      "authors": [
        "Fu Teng",
        "Miao Pan",
        "Xuhong Zhang",
        "Zhezhi He",
        "Yiyao Yang",
        "Xinyi Chai",
        "Mengnan Qi",
        "Liqiang Lu",
        "Jianwei Yin"
      ],
      "summary": "Recent advancements in code generation have shown remarkable success across software domains, yet hardware description languages (HDLs) such as Verilog remain underexplored due to their concurrency semantics, syntactic rigidity, and simulation complexity. In this work, we address these challenges by introducing a reinforcement learning (RL) framework tailored for Verilog code generation. We first construct Veribench-53K, a high-quality dataset curated from over 700K Verilog problems, enriched with structured prompts, complexity labels, and diverse testbenches. To tackle the problem of sparse and noisy reward signals, we propose a Trace-back based Rescore mechanism that leverages reasoning paths and iterative refinement to enhance feedback reliability and support reward model training. Furthermore, to mitigate catastrophic forgetting and overfitting during RL fine-tuning, we introduce a sample-balanced weighting strategy that adaptively balances learning dynamics based on reward-probability distributions. These innovations are integrated into an iterative RL pipeline that co-evolves the policy and reward models. In contrast to recent work such as CraftRTL, which relies on large-scale closed-source model distillation, and DeepSeek-style approaches that struggle with sparse feedback, our method demonstrates superior performance using a smaller but high-quality dataset combined with RL optimization. Experiments on Verilog generation tasks demonstrate state-of-the-art performance, with substantial gains in test pass rate, functional correctness, and compilation robustness. Our findings highlight the potential of RL-driven approaches for structured code generation in hardware-centric domains. VERIRL is publicly available at https://github.com/omniAI-Lab/VeriRL.",
      "arxiv_id": "2508.18462v1",
      "published": "2025-08-25",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.18462v1",
      "pdf_url": "https://arxiv.org/pdf/2508.18462v1.pdf"
    },
    {
      "title": "Segmenting Text and Learning Their Rewards for Improved RLHF in Language Model",
      "authors": [
        "Yueqin Yin",
        "Shentao Yang",
        "Yujia Xie",
        "Ziyi Yang",
        "Yuting Sun",
        "Hany Awadalla",
        "Weizhu Chen",
        "Mingyuan Zhou"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has been widely adopted to align language models (LMs) with human preference. Prior RLHF works typically take a bandit formulation, which, though intuitive, ignores the sequential nature of LM generation and can suffer from the sparse reward issue. While recent works propose dense token-level RLHF, treating each token as an action may be oversubtle to proper reward assignment. In this paper, we seek to get the best of both by training and utilizing a segment-level reward model, which assigns a reward to each semantically complete text segment that spans over a short sequence of tokens. For reward learning, our method allows dynamic text segmentation and compatibility with standard sequence-preference datasets. For effective RL-based LM training against segment reward, we generalize the classical scalar bandit reward normalizers into location-aware normalizer functions and interpolate the segment reward for further densification. With these designs, our method performs competitively on three popular RLHF benchmarks for LM policy: AlpacaEval 2.0, Arena-Hard, and MT-Bench. Ablation studies are conducted to further demonstrate our method.",
      "arxiv_id": "2501.02790v1",
      "published": "2025-01-06",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2501.02790v1",
      "pdf_url": "https://arxiv.org/pdf/2501.02790v1.pdf"
    },
    {
      "title": "KL-Regularised Q-Learning: A Token-level Action-Value perspective on Online RLHF",
      "authors": [
        "Jason R Brown",
        "Lennie Wells",
        "Edward James Young",
        "Sergio Bacallado"
      ],
      "summary": "Proximal Policy Optimisation (PPO) is an established and effective policy gradient algorithm used for Language Model Reinforcement Learning from Human Feedback (LM-RLHF). PPO performs well empirically but has a heuristic motivation and handles the KL-divergence constraint used in LM-RLHF in an ad-hoc manner. In this paper, we develop a a new action-value RL method for the LM-RLHF setting, KL-regularised Q-Learning (KLQ). We then show that our method is equivalent to a version of PPO in a certain specific sense, despite its very different motivation. Finally, we benchmark KLQ on two key language generation tasks -- summarisation and single-turn dialogue. We demonstrate that KLQ performs on-par with PPO at optimising the LM-RLHF objective, and achieves a consistently higher win-rate against PPO on LLM-as-a-judge evaluations.",
      "arxiv_id": "2508.17000v1",
      "published": "2025-08-23",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2508.17000v1",
      "pdf_url": "https://arxiv.org/pdf/2508.17000v1.pdf"
    },
    {
      "title": "Understanding and Alleviating Memory Consumption in RLHF for LLMs",
      "authors": [
        "Jin Zhou",
        "Hanmei Yang",
        " Steven",
        " Tang",
        "Mingcan Xiang",
        "Hui Guan",
        "Tongping Liu"
      ],
      "summary": "Fine-tuning with Reinforcement Learning with Human Feedback (RLHF) is essential for aligning large language models (LLMs). However, RLHF often encounters significant memory challenges. This study is the first to examine memory usage in the RLHF context, exploring various memory management strategies and unveiling the reasons behind excessive memory consumption. Additionally, we introduce a simple yet effective approach that substantially reduces the memory required for RLHF fine-tuning.",
      "arxiv_id": "2410.15651v1",
      "published": "2024-10-21",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.15651v1",
      "pdf_url": "https://arxiv.org/pdf/2410.15651v1.pdf"
    },
    {
      "title": "Breaking the SFT Plateau: Multimodal Structured Reinforcement Learning for Chart-to-Code Generation",
      "authors": [
        "Lei Chen",
        "Xuanle Zhao",
        "Zhixiong Zeng",
        "Jing Huang",
        "Liming Zheng",
        "Yufeng Zhong",
        "Lin Ma"
      ],
      "summary": "While reinforcement learning (RL) has proven highly effective for general reasoning in vision-language models, its application to tasks requiring in-depth understanding of information-rich images and generation of structured outputs remains underexplored. Chart-to-code generation exemplifies this challenge, demanding complex reasoning over visual charts to generate structured code. Supervised fine-tuning (SFT) alone is often insufficient, highlighting the need for effective RL strategies that appropriately reward structured outputs. We systematically investigate the performance plateau in SFT through large-scale experiments and propose Multimodal Structured Reinforcement Learning (MSRL) for chart-to-code generation, which substantially breaks through this plateau. We construct the largest training corpus to date, containing 3 million chart-code pairs from real-world arXiv tables to mitigate simplistic patterns of prior synthetic data. Despite reaching state-of-the-art performance, our experiments show that scaling SFT data eventually hits a plateau where further increases yield negligible improvements. Our MSRL method leverages a multi-granularity structured reward system using multimodal textual and visual feedback. At the textual level, rule-based rewards validate fine-grained code details. At the visual level, model-based rewards assess structural similarity by rendering generated code into images and employing an evaluator model. We implement this within a two-stage curriculum for training stability. Results demonstrate that MSRL significantly breaks the SFT plateau, improving high-level metrics by 6.2% and 9.9% on ChartMimic and ReachQA benchmarks respectively, achieving competitive performance with advanced closed-source models.",
      "arxiv_id": "2508.13587v1",
      "published": "2025-08-19",
      "categories": [
        "cs.AI",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2508.13587v1",
      "pdf_url": "https://arxiv.org/pdf/2508.13587v1.pdf"
    },
    {
      "title": "AlgoPilot: Fully Autonomous Program Synthesis Without Human-Written Programs",
      "authors": [
        "Xiaoxin Yin"
      ],
      "summary": "Program synthesis has traditionally relied on human-provided specifications, examples, or prior knowledge to generate functional algorithms. Existing methods either emulate human-written algorithms or solve specific tasks without generating reusable programmatic logic, limiting their ability to create novel algorithms. We introduce AlgoPilot, a groundbreaking approach for fully automated program synthesis without human-written programs or trajectories. AlgoPilot leverages reinforcement learning (RL) guided by a Trajectory Language Model (TLM) to synthesize algorithms from scratch. The TLM, trained on trajectories generated by random Python functions, serves as a soft constraint during the RL process, aligning generated sequences with patterns likely to represent valid algorithms. Using sorting as a test case, AlgoPilot demonstrates its ability to generate trajectories that are interpretable as classical algorithms, such as Bubble Sort, while operating without prior algorithmic knowledge. This work establishes a new paradigm for algorithm discovery and lays the groundwork for future advancements in autonomous program synthesis.",
      "arxiv_id": "2501.06423v1",
      "published": "2025-01-11",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2501.06423v1",
      "pdf_url": "https://arxiv.org/pdf/2501.06423v1.pdf"
    },
    {
      "title": "Enhancing RLHF with Human Gaze Modeling",
      "authors": [
        "Karim Galliamov",
        "Ivan Titov",
        "Ilya Pershin"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) aligns language models with human preferences but is computationally expensive. We explore two approaches that leverage human gaze modeling to enhance RLHF: (1) gaze-aware reward models and (2) gaze-based distribution of sparse rewards at token level. Our experiments demonstate that gaze-informed RLHF achieves faster convergence while maintaining or slightly improving performance, thus, reducing computational costs during policy optimization. These results show that human gaze provides a valuable and underused signal for policy optimization, pointing to a promising direction for improving RLHF efficiency.",
      "arxiv_id": "2507.09016v2",
      "published": "2025-07-11",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2507.09016v2",
      "pdf_url": "https://arxiv.org/pdf/2507.09016v2.pdf"
    },
    {
      "title": "HERO: Human-Feedback Efficient Reinforcement Learning for Online Diffusion Model Finetuning",
      "authors": [
        "Ayano Hiranaka",
        "Shang-Fu Chen",
        "Chieh-Hsin Lai",
        "Dongjun Kim",
        "Naoki Murata",
        "Takashi Shibuya",
        "Wei-Hsiang Liao",
        "Shao-Hua Sun",
        "Yuki Mitsufuji"
      ],
      "summary": "Controllable generation through Stable Diffusion (SD) fine-tuning aims to improve fidelity, safety, and alignment with human guidance. Existing reinforcement learning from human feedback methods usually rely on predefined heuristic reward functions or pretrained reward models built on large-scale datasets, limiting their applicability to scenarios where collecting such data is costly or difficult. To effectively and efficiently utilize human feedback, we develop a framework, HERO, which leverages online human feedback collected on the fly during model learning. Specifically, HERO features two key mechanisms: (1) Feedback-Aligned Representation Learning, an online training method that captures human feedback and provides informative learning signals for fine-tuning, and (2) Feedback-Guided Image Generation, which involves generating images from SD's refined initialization samples, enabling faster convergence towards the evaluator's intent. We demonstrate that HERO is 4x more efficient in online feedback for body part anomaly correction compared to the best existing method. Additionally, experiments show that HERO can effectively handle tasks like reasoning, counting, personalization, and reducing NSFW content with only 0.5K online feedback. The code and project page are available at https://hero-dm.github.io/.",
      "arxiv_id": "2410.05116v3",
      "published": "2024-10-07",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CV",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2410.05116v3",
      "pdf_url": "https://arxiv.org/pdf/2410.05116v3.pdf"
    },
    {
      "title": "A Survey of Reinforcement Learning from Human Feedback",
      "authors": [
        "Timo Kaufmann",
        "Paul Weng",
        "Viktor Bengs",
        "Eyke H\u00fcllermeier"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is a variant of reinforcement learning (RL) that learns from human feedback instead of relying on an engineered reward function. Building on prior work on the related setting of preference-based reinforcement learning (PbRL), it stands at the intersection of artificial intelligence and human-computer interaction. This positioning offers a promising avenue to enhance the performance and adaptability of intelligent systems while also improving the alignment of their objectives with human values. The training of large language models (LLMs) has impressively demonstrated this potential in recent years, where RLHF played a decisive role in directing the model's capabilities toward human objectives. This article provides a comprehensive overview of the fundamentals of RLHF, exploring the intricate dynamics between RL agents and human input. While recent focus has been on RLHF for LLMs, our survey adopts a broader perspective, examining the diverse applications and wide-ranging impact of the technique. We delve into the core principles that underpin RLHF, shedding light on the symbiotic relationship between algorithms and human feedback, and discuss the main research trends in the field. By synthesizing the current landscape of RLHF research, this article aims to provide researchers as well as practitioners with a comprehensive understanding of this rapidly growing field of research.",
      "arxiv_id": "2312.14925v2",
      "published": "2023-12-22",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2312.14925v2",
      "pdf_url": "https://arxiv.org/pdf/2312.14925v2.pdf"
    },
    {
      "title": "Reward Model Overoptimisation in Iterated RLHF",
      "authors": [
        "Lorenz Wolf",
        "Robert Kirk",
        "Mirco Musolesi"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is a widely used method for aligning large language models with human preferences. However, RLHF often suffers from reward model overoptimisation, in which models overfit to the reward function, resulting in non-generalisable policies that exploit the idiosyncrasies and peculiarities of the reward function. A common mitigation is iterated RLHF, in which reward models are repeatedly retrained with updated human feedback and policies are re-optimised. Despite its increasing adoption, the dynamics of overoptimisation in this setting remain poorly understood. In this work, we present the first comprehensive study of overoptimisation in iterated RLHF. We systematically analyse key design choices - how reward model training data is transferred across iterations, which reward function is used for optimisation, and how policies are initialised. Using the controlled AlpacaFarm benchmark, we observe that overoptimisation tends to decrease over successive iterations, as reward models increasingly approximate ground-truth preferences. However, performance gains diminish over time, and while reinitialising from the base policy is robust, it limits optimisation flexibility. Other initialisation strategies often fail to recover from early overoptimisation. These findings offer actionable insights for building more stable and generalisable RLHF pipelines.",
      "arxiv_id": "2505.18126v2",
      "published": "2025-05-23",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.18126v2",
      "pdf_url": "https://arxiv.org/pdf/2505.18126v2.pdf"
    },
    {
      "title": "Value-Incentivized Preference Optimization: A Unified Approach to Online and Offline RLHF",
      "authors": [
        "Shicong Cen",
        "Jincheng Mei",
        "Katayoon Goshvadi",
        "Hanjun Dai",
        "Tong Yang",
        "Sherry Yang",
        "Dale Schuurmans",
        "Yuejie Chi",
        "Bo Dai"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has demonstrated great promise in aligning large language models (LLMs) with human preference. Depending on the availability of preference data, both online and offline RLHF are active areas of investigation. A key bottleneck is understanding how to incorporate uncertainty estimation in the reward function learned from the preference data for RLHF, regardless of how the preference data is collected. While the principles of optimism or pessimism under uncertainty are well-established in standard reinforcement learning (RL), a practically-implementable and theoretically-grounded form amenable to large language models is not yet available, as standard techniques for constructing confidence intervals become intractable under arbitrary policy parameterizations.   In this paper, we introduce a unified approach to online and offline RLHF -- value-incentivized preference optimization (VPO) -- which regularizes the maximum-likelihood estimate of the reward function with the corresponding value function, modulated by a $\\textit{sign}$ to indicate whether the optimism or pessimism is chosen. VPO also directly optimizes the policy with implicit reward modeling, and therefore shares a simpler RLHF pipeline similar to direct preference optimization. Theoretical guarantees of VPO are provided for both online and offline settings, matching the rates of their standard RL counterparts. Moreover, experiments on text summarization and dialog verify the practicality and effectiveness of VPO.",
      "arxiv_id": "2405.19320v4",
      "published": "2024-05-29",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2405.19320v4",
      "pdf_url": "https://arxiv.org/pdf/2405.19320v4.pdf"
    },
    {
      "title": "Online Bandit Learning with Offline Preference Data for Improved RLHF",
      "authors": [
        "Akhil Agnihotri",
        "Rahul Jain",
        "Deepak Ramachandran",
        "Zheng Wen"
      ],
      "summary": "Reinforcement Learning with Human Feedback (RLHF) is at the core of fine-tuning methods for generative AI models for language and images. Such feedback is often sought as rank or preference feedback from human raters, as opposed to eliciting scores since the latter tends to be noisy. On the other hand, RL theory and algorithms predominantly assume that a reward feedback is available. In particular, approaches for online learning that can be helpful in adaptive data collection via active learning cannot incorporate offline preference data. In this paper, we adopt a finite-armed linear bandit model as a prototypical model of online learning. We consider an offline preference dataset to be available generated by an expert of unknown 'competence'. We propose warmPref-PS, a posterior sampling algorithm for online learning that can be warm-started with an offline dataset with noisy preference feedback. We show that by modeling the 'competence' of the expert that generated it, we are able to use such a dataset most effectively. We support our claims with novel theoretical analysis of its Bayesian regret, as well as, extensive empirical evaluation of an approximate loss function that optimizes for infinitely many arms, and performs substantially better than baselines.",
      "arxiv_id": "2406.09574v4",
      "published": "2024-06-13",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2406.09574v4",
      "pdf_url": "https://arxiv.org/pdf/2406.09574v4.pdf"
    },
    {
      "title": "Mapping Social Choice Theory to RLHF",
      "authors": [
        "Jessica Dai",
        "Eve Fleisig"
      ],
      "summary": "Recent work on the limitations of using reinforcement learning from human feedback (RLHF) to incorporate human preferences into model behavior often raises social choice theory as a reference point. Social choice theory's analysis of settings such as voting mechanisms provides technical infrastructure that can inform how to aggregate human preferences amid disagreement. We analyze the problem settings of social choice and RLHF, identify key differences between them, and discuss how these differences may affect the RLHF interpretation of well-known technical results in social choice.",
      "arxiv_id": "2404.13038v1",
      "published": "2024-04-19",
      "categories": [
        "cs.AI",
        "cs.CY"
      ],
      "url": "https://arxiv.org/abs/2404.13038v1",
      "pdf_url": "https://arxiv.org/pdf/2404.13038v1.pdf"
    },
    {
      "title": "ChartMaster: Advancing Chart-to-Code Generation with Real-World Charts and Chart Similarity Reinforcement Learning",
      "authors": [
        "Wentao Tan",
        "Qiong Cao",
        "Chao Xue",
        "Yibing Zhan",
        "Changxing Ding",
        "Xiaodong He"
      ],
      "summary": "The chart-to-code generation task requires MLLMs to convert chart images into executable code. This task faces two main challenges: limited data diversity and the difficulty of maintaining visual consistency between generated charts and the original ones. Existing datasets mainly rely on synthetic seed data to prompt GPT models for code generation, resulting in homogeneous samples that limit model generalization to real-world chart styles. To address this, we propose ReChartPrompt, leveraging real-world, human-designed charts extracted from arXiv papers as prompts. By harnessing the rich content and diverse visual styles of arXiv charts, we construct ReChartPrompt-240K, a large-scale and highly diverse dataset that better reflects realistic chart variations. For the second challenge, although SFT improves code understanding by optimizing next-token prediction, it does not provide direct supervision on visual features. As a result, it often fails to guarantee that the generated charts visually match the original ones. To address this, we propose ChartSimRL, a GRPO-based reinforcement learning algorithm guided by a novel chart similarity reward. This reward consists of two components: attribute similarity, which measures the overlap of chart attributes like layout and color between the generated and original charts, and visual similarity, which evaluates overall visual features, including texture, using convolutional neural networks. Unlike traditional text-based rewards, our reward accounts for the multimodal nature of the chart-to-code generation task, significantly enhancing the model's ability to accurately reproduce charts. Integrating ReChartPrompt and ChartSimRL, we develop the ChartMaster model, achieving SOTA results among 7B-parameter models and rivaling GPT-4o on various chart-to-code benchmarks. All resources are available at https://github.com/WentaoTan/ChartMaster.",
      "arxiv_id": "2508.17608v2",
      "published": "2025-08-25",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2508.17608v2",
      "pdf_url": "https://arxiv.org/pdf/2508.17608v2.pdf"
    },
    {
      "title": "FALCON: Feedback-driven Adaptive Long/short-term memory reinforced Coding Optimization system",
      "authors": [
        "Zeyuan Li",
        "Yangfan He",
        "Lewei He",
        "Jianhui Wang",
        "Tianyu Shi",
        "Bin Lei",
        "Yuchen Li",
        "Qiuwu Chen"
      ],
      "summary": "Recently, large language models (LLMs) have achieved significant progress in automated code generation. Despite their strong instruction-following capabilities, these models frequently struggled to align with user intent in coding scenarios. In particular, they were hampered by datasets that lacked diversity and failed to address specialized tasks or edge cases. Furthermore, challenges in supervised fine-tuning (SFT) and reinforcement learning from human feedback (RLHF) led to failures in generating precise, human-intent-aligned code. To tackle these challenges and improve the code generation performance for automated programming systems, we propose Feedback-driven Adaptive Long/short-term memory reinforced Coding Optimization (i.e., FALCON). FALCON is structured into two hierarchical levels. From the global level, long-term memory improves code quality by retaining and applying learned knowledge. At the local level, short-term memory allows for the incorporation of immediate feedback from compilers and AI systems. Additionally, we introduce meta-reinforcement learning with feedback rewards to solve the global-local bi-level optimization problem and enhance the model's adaptability across diverse code generation tasks. Extensive experiments demonstrate that our technique achieves state-of-the-art performance, leading other reinforcement learning methods by more than 4.5 percentage points on the MBPP benchmark and 6.1 percentage points on the Humaneval benchmark. The open-sourced code is publicly available at https://github.com/titurte/FALCON.",
      "arxiv_id": "2410.21349v5",
      "published": "2024-10-28",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.PF"
      ],
      "url": "https://arxiv.org/abs/2410.21349v5",
      "pdf_url": "https://arxiv.org/pdf/2410.21349v5.pdf"
    },
    {
      "title": "Tidal MerzA: Combining affective modelling and autonomous code generation through Reinforcement Learning",
      "authors": [
        "Elizabeth Wilson",
        "Gy\u00f6rgy Fazekas",
        "Geraint Wiggins"
      ],
      "summary": "This paper presents Tidal-MerzA, a novel system designed for collaborative performances between humans and a machine agent in the context of live coding, specifically focusing on the generation of musical patterns. Tidal-MerzA fuses two foundational models: ALCAA (Affective Live Coding Autonomous Agent) and Tidal Fuzz, a computational framework. By integrating affective modelling with computational generation, this system leverages reinforcement learning techniques to dynamically adapt music composition parameters within the TidalCycles framework, ensuring both affective qualities to the patterns and syntactical correctness. The development of Tidal-MerzA introduces two distinct agents: one focusing on the generation of mini-notation strings for musical expression, and another on the alignment of music with targeted affective states through reinforcement learning. This approach enhances the adaptability and creative potential of live coding practices and allows exploration of human-machine creative interactions. Tidal-MerzA advances the field of computational music generation, presenting a novel methodology for incorporating artificial intelligence into artistic practices.",
      "arxiv_id": "2409.07918v1",
      "published": "2024-09-12",
      "categories": [
        "cs.HC",
        "cs.AI",
        "cs.SD",
        "eess.AS"
      ],
      "url": "https://arxiv.org/abs/2409.07918v1",
      "pdf_url": "https://arxiv.org/pdf/2409.07918v1.pdf"
    },
    {
      "title": "Generating Code World Models with Large Language Models Guided by Monte Carlo Tree Search",
      "authors": [
        "Nicola Dainese",
        "Matteo Merler",
        "Minttu Alakuijala",
        "Pekka Marttinen"
      ],
      "summary": "In this work we consider Code World Models, world models generated by a Large Language Model (LLM) in the form of Python code for model-based Reinforcement Learning (RL). Calling code instead of LLMs for planning has potential to be more precise, reliable, interpretable, and extremely efficient. However, writing appropriate Code World Models requires the ability to understand complex instructions, to generate exact code with non-trivial logic and to self-debug a long program with feedback from unit tests and environment trajectories. To address these challenges, we propose Generate, Improve and Fix with Monte Carlo Tree Search (GIF-MCTS), a new code generation strategy for LLMs. To test our approach in an offline RL setting, we introduce the Code World Models Benchmark (CWMB), a suite of program synthesis and planning tasks comprised of 18 diverse RL environments paired with corresponding textual descriptions and curated trajectories. GIF-MCTS surpasses all baselines on the CWMB and two other benchmarks, and we show that the Code World Models synthesized with it can be successfully used for planning, resulting in model-based RL agents with greatly improved sample efficiency and inference speed.",
      "arxiv_id": "2405.15383v2",
      "published": "2024-05-24",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2405.15383v2",
      "pdf_url": "https://arxiv.org/pdf/2405.15383v2.pdf"
    },
    {
      "title": "Accelerating RLHF Training with Reward Variance Increase",
      "authors": [
        "Zonglin Yang",
        "Zhexuan Gu",
        "Houduo Qi",
        "Yancheng Yuan"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is an essential technique for ensuring that large language models (LLMs) are aligned with human values and preferences during the post-training phase. As an effective RLHF approach, group relative policy optimization (GRPO) has demonstrated success in many LLM-based applications. However, efficient GRPO-based RLHF training remains a challenge. Recent studies reveal that a higher reward variance of the initial policy model leads to faster RLHF training. Inspired by this finding, we propose a practical reward adjustment model to accelerate RLHF training by provably increasing the reward variance and preserving the relative preferences and reward expectation. Our reward adjustment method inherently poses a nonconvex optimization problem, which is NP-hard to solve in general. To overcome the computational challenges, we design a novel $O(n \\log n)$ algorithm to find a global solution of the nonconvex reward adjustment model by explicitly characterizing the extreme points of the feasible set. As an important application, we naturally integrate this reward adjustment model into the GRPO algorithm, leading to a more efficient GRPO with reward variance increase (GRPOVI) algorithm for RLHF training. As an interesting byproduct, we provide an indirect explanation for the empirical effectiveness of GRPO with rule-based reward for RLHF training, as demonstrated in DeepSeek-R1. Experiment results demonstrate that the GRPOVI algorithm can significantly improve the RLHF training efficiency compared to the original GRPO algorithm.",
      "arxiv_id": "2505.23247v2",
      "published": "2025-05-29",
      "categories": [
        "cs.LG",
        "cs.AI",
        "math.OC"
      ],
      "url": "https://arxiv.org/abs/2505.23247v2",
      "pdf_url": "https://arxiv.org/pdf/2505.23247v2.pdf"
    },
    {
      "title": "Exploratory Preference Optimization: Harnessing Implicit Q*-Approximation for Sample-Efficient RLHF",
      "authors": [
        "Tengyang Xie",
        "Dylan J. Foster",
        "Akshay Krishnamurthy",
        "Corby Rosset",
        "Ahmed Awadallah",
        "Alexander Rakhlin"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has emerged as a central tool for language model alignment. We consider online exploration in RLHF, which exploits interactive access to human or AI feedback by deliberately encouraging the model to produce diverse, maximally informative responses. By allowing RLHF to confidently stray from the pre-trained model, online exploration offers the possibility of novel, potentially super-human capabilities, but its full potential as a paradigm for language model training has yet to be realized, owing to computational and statistical bottlenecks in directly adapting existing reinforcement learning techniques. We propose a new algorithm for online exploration in RLHF, Exploratory Preference Optimization (XPO), which is simple and practical -- a one-line change to (online) Direct Preference Optimization (DPO; Rafailov et al., 2023) -- yet enjoys the strongest known provable guarantees and promising empirical performance. XPO augments the DPO objective with a novel and principled exploration bonus, empowering the algorithm to explore outside the support of the initial model and human feedback data. In theory, we show that XPO is provably sample-efficient and converges to a near-optimal language model policy under natural exploration conditions, irrespective of whether the initial model has good coverage. Our analysis, which builds on the observation that DPO implicitly performs a form of $Q^{\\star}$-approximation (or, Bellman error minimization), combines previously disparate techniques from language modeling and theoretical reinforcement learning in a serendipitous fashion through the perspective of KL-regularized Markov decision processes. Empirically, we find that XPO is more sample-efficient than non-exploratory DPO variants in a preliminary evaluation.",
      "arxiv_id": "2405.21046v1",
      "published": "2024-05-31",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2405.21046v1",
      "pdf_url": "https://arxiv.org/pdf/2405.21046v1.pdf"
    },
    {
      "title": "Residual Skill Policies: Learning an Adaptable Skill-based Action Space for Reinforcement Learning for Robotics",
      "authors": [
        "Krishan Rana",
        "Ming Xu",
        "Brendan Tidd",
        "Michael Milford",
        "Niko S\u00fcnderhauf"
      ],
      "summary": "Skill-based reinforcement learning (RL) has emerged as a promising strategy to leverage prior knowledge for accelerated robot learning. Skills are typically extracted from expert demonstrations and are embedded into a latent space from which they can be sampled as actions by a high-level RL agent. However, this skill space is expansive, and not all skills are relevant for a given robot state, making exploration difficult. Furthermore, the downstream RL agent is limited to learning structurally similar tasks to those used to construct the skill space. We firstly propose accelerating exploration in the skill space using state-conditioned generative models to directly bias the high-level agent towards only sampling skills relevant to a given state based on prior experience. Next, we propose a low-level residual policy for fine-grained skill adaptation enabling downstream RL agents to adapt to unseen task variations. Finally, we validate our approach across four challenging manipulation tasks that differ from those used to build the skill space, demonstrating our ability to learn across task variations while significantly accelerating exploration, outperforming prior works. Code and videos are available on our project website: https://krishanrana.github.io/reskill.",
      "arxiv_id": "2211.02231v1",
      "published": "2022-11-04",
      "categories": [
        "cs.RO",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2211.02231v1",
      "pdf_url": "https://arxiv.org/pdf/2211.02231v1.pdf"
    },
    {
      "title": "One Framework to Rule Them All: Unifying RL-Based and RL-Free Methods in RLHF",
      "authors": [
        "Xin Cai"
      ],
      "summary": "In this article, we primarily examine a variety of RL-based and RL-free methods designed to address Reinforcement Learning from Human Feedback (RLHF) and Large Reasoning Models (LRMs). We begin with a concise overview of the typical steps involved in RLHF and LRMs. Next, we reinterpret several RL-based and RL-free algorithms through the perspective of neural structured bandit prediction, providing a clear conceptual framework that uncovers a deeper connection between these seemingly distinct approaches. Following this, we briefly review some core principles of reinforcement learning, drawing attention to an often-overlooked aspect in existing RLHF studies. This leads to a detailed derivation of the standard RLHF objective within a full RL context, demonstrating its equivalence to neural structured bandit prediction. Finally, by reinvestigating the principles behind Proximal Policy Optimization (PPO), we pinpoint areas needing adjustment, which culminates in the introduction of the Generalized Reinforce Optimization (GRO) framework, seamlessly integrating RL-based and RL-free methods in RLHF. We look forward to the community's efforts to empirically validate GRO and invite constructive feedback.",
      "arxiv_id": "2503.19523v2",
      "published": "2025-03-25",
      "categories": [
        "cs.LG",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2503.19523v2",
      "pdf_url": "https://arxiv.org/pdf/2503.19523v2.pdf"
    },
    {
      "title": "CompilerDream: Learning a Compiler World Model for General Code Optimization",
      "authors": [
        "Chaoyi Deng",
        "Jialong Wu",
        "Ningya Feng",
        "Jianmin Wang",
        "Mingsheng Long"
      ],
      "summary": "Effective code optimization in compilers is crucial for computer and software engineering. The success of these optimizations primarily depends on the selection and ordering of the optimization passes applied to the code. While most compilers rely on a fixed sequence of optimization passes, current methods to find the optimal sequence either employ impractically slow search algorithms or learning methods that struggle to generalize to code unseen during training. We introduce CompilerDream, a model-based reinforcement learning approach to general code optimization. CompilerDream comprises a compiler world model that accurately simulates the intrinsic properties of optimization passes and an agent trained on this model to produce effective optimization strategies. By training on a large-scale program dataset, CompilerDream is equipped to serve as a general code optimizer across various application scenarios and source-code languages. Our extensive experiments first highlight CompilerDream's strong optimization capabilities for autotuning, where it leads the CompilerGym leaderboard. More importantly, the zero-shot generalization ability of large-scale trained compiler world model and agent, excels across diverse datasets, surpassing LLVM's built-in optimizations and other state-of-the-art methods in both settings of value prediction and end-to-end code optimization.",
      "arxiv_id": "2404.16077v3",
      "published": "2024-04-24",
      "categories": [
        "cs.PL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2404.16077v3",
      "pdf_url": "https://arxiv.org/pdf/2404.16077v3.pdf"
    },
    {
      "title": "A Unified Pairwise Framework for RLHF: Bridging Generative Reward Modeling and Policy Optimization",
      "authors": [
        "Wenyuan Xu",
        "Xiaochen Zuo",
        "Chao Xin",
        "Yu Yue",
        "Lin Yan",
        "Yonghui Wu"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a important paradigm for aligning large language models (LLMs) with human preferences during post-training. This framework typically involves two stages: first, training a reward model on human preference data, followed by optimizing the language model using reinforcement learning algorithms. However, current RLHF approaches may constrained by two limitations. First, existing RLHF frameworks often rely on Bradley-Terry models to assign scalar rewards based on pairwise comparisons of individual responses. However, this approach imposes significant challenges on reward model (RM), as the inherent variability in prompt-response pairs across different contexts demands robust calibration capabilities from the RM. Second, reward models are typically initialized from generative foundation models, such as pre-trained or supervised fine-tuned models, despite the fact that reward models perform discriminative tasks, creating a mismatch. This paper introduces Pairwise-RL, a RLHF framework that addresses these challenges through a combination of generative reward modeling and a pairwise proximal policy optimization (PPO) algorithm. Pairwise-RL unifies reward model training and its application during reinforcement learning within a consistent pairwise paradigm, leveraging generative modeling techniques to enhance reward model performance and score calibration. Experimental evaluations demonstrate that Pairwise-RL outperforms traditional RLHF frameworks across both internal evaluation datasets and standard public benchmarks, underscoring its effectiveness in improving alignment and model behavior.",
      "arxiv_id": "2504.04950v1",
      "published": "2025-04-07",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.04950v1",
      "pdf_url": "https://arxiv.org/pdf/2504.04950v1.pdf"
    },
    {
      "title": "A Theoretical Framework for Partially Observed Reward-States in RLHF",
      "authors": [
        "Chinmaya Kausik",
        "Mirco Mutti",
        "Aldo Pacchiano",
        "Ambuj Tewari"
      ],
      "summary": "The growing deployment of reinforcement learning from human feedback (RLHF) calls for a deeper theoretical investigation of its underlying models. The prevalent models of RLHF do not account for neuroscience-backed, partially-observed \"internal states\" that can affect human feedback, nor do they accommodate intermediate feedback during an interaction. Both of these can be instrumental in speeding up learning and improving alignment. To address these limitations, we model RLHF as reinforcement learning with partially observed reward-states (PORRL). We accommodate two kinds of feedback $-$ cardinal and dueling feedback. We first demonstrate that PORRL subsumes a wide class of RL problems, including traditional RL, RLHF, and reward machines. For cardinal feedback, we present two model-based methods (POR-UCRL, POR-UCBVI). We give both cardinal regret and sample complexity guarantees for the methods, showing that they improve over naive history-summarization. We then discuss the benefits of a model-free method like GOLF with naive history-summarization in settings with recursive internal states and dense intermediate feedback. For this purpose, we define a new history aware version of the Bellman-eluder dimension and give a new guarantee for GOLF in our setting, which can be exponentially sharper in illustrative examples. For dueling feedback, we show that a naive reduction to cardinal feedback fails to achieve sublinear dueling regret. We then present the first explicit reduction that converts guarantees for cardinal regret to dueling regret. In both feedback settings, we show that our models and guarantees generalize and extend existing ones.",
      "arxiv_id": "2402.03282v3",
      "published": "2024-02-05",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2402.03282v3",
      "pdf_url": "https://arxiv.org/pdf/2402.03282v3.pdf"
    },
    {
      "title": "The Perfect Blend: Redefining RLHF with Mixture of Judges",
      "authors": [
        "Tengyu Xu",
        "Eryk Helenowski",
        "Karthik Abinav Sankararaman",
        "Di Jin",
        "Kaiyan Peng",
        "Eric Han",
        "Shaoliang Nie",
        "Chen Zhu",
        "Hejia Zhang",
        "Wenxuan Zhou",
        "Zhouhao Zeng",
        "Yun He",
        "Karishma Mandyam",
        "Arya Talabzadeh",
        "Madian Khabsa",
        "Gabriel Cohen",
        "Yuandong Tian",
        "Hao Ma",
        "Sinong Wang",
        "Han Fang"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has become the leading approach for fine-tuning large language models (LLM). However, RLHF has limitations in multi-task learning (MTL) due to challenges of reward hacking and extreme multi-objective optimization (i.e., trade-off of multiple and/or sometimes conflicting objectives). Applying RLHF for MTL currently requires careful tuning of the weights for reward model and data combinations. This is often done via human intuition and does not generalize. In this work, we introduce a novel post-training paradigm which we called Constrained Generative Policy Optimization (CGPO). The core of CGPO is Mixture of Judges (MoJ) with cost-efficient constrained policy optimization with stratification, which can identify the perfect blend in RLHF in a principled manner. It shows strong empirical results with theoretical guarantees, does not require extensive hyper-parameter tuning, and is plug-and-play in common post-training pipelines. Together, this can detect and mitigate reward hacking behaviors while reaching a pareto-optimal point across an extremely large number of objectives.   Our empirical evaluations demonstrate that CGPO significantly outperforms standard RLHF algorithms like PPO and DPO across various tasks including general chat, STEM questions, instruction following, and coding. Specifically, CGPO shows improvements of 7.4% in AlpacaEval-2 (general chat), 12.5% in Arena-Hard (STEM & reasoning), and consistent gains in other domains like math and coding. Notably, PPO, while commonly used, is prone to severe reward hacking in popular coding benchmarks, which CGPO successfully addresses. This breakthrough in RLHF not only tackles reward hacking and extreme multi-objective optimization challenges but also advances the state-of-the-art in aligning general-purpose LLMs for diverse applications.",
      "arxiv_id": "2409.20370v1",
      "published": "2024-09-30",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2409.20370v1",
      "pdf_url": "https://arxiv.org/pdf/2409.20370v1.pdf"
    },
    {
      "title": "An Adaptive Placement and Parallelism Framework for Accelerating RLHF Training",
      "authors": [
        "Youshao Xiao",
        "Zhenglei Zhou",
        "Fagui Mao",
        "Weichang Wu",
        "Shangchun Zhao",
        "Lin Ju",
        "Lei Liang",
        "Xiaolu Zhang",
        "Jun Zhou"
      ],
      "summary": "Recently, ChatGPT or InstructGPT like large language models (LLM) has made a significant impact in the AI world. Many works have attempted to reproduce the complex InstructGPT's training pipeline, namely Reinforcement Learning with Human Feedback (RLHF). However, the mainstream distributed RLHF training methods typically adopt a fixed model placement strategy, referred to as the Co-located strategy. This strategy treats all four interdependent models involved in RLHF as a single entity, distributing them across all devices and applying parallelism techniques designed for a single model, regardless of the workload heterogeneity inherent to each model. As a result, this strategy exacerbates the generation bottlenecks in the RLHF training and degrades the overall training efficiency. To address these issues, we propose a flexible model placement framework that offers two general and agile model placement strategies. The Interleaving strategy helps reduce memory redundancy and communication costs of RLHF training by placing models without dependencies on exclusive devices with careful orchestration. On the other hand, the Disaggregated strategy improves the throughput of model training by separating the training and inference runtime of the RLHF pipeline with additional shadow models. Furthermore, our framework provides a simple user interface and guidelines to easily and flexibly configure these strategies in various training scenarios. Our experiments have shown that our strategy can achieve notable improvements up to 11x, compared to the current state-of-the-art (SOTA) approaches. The results highlight the effectiveness and adaptability of our methods in accelerating the training of distributed RLHF.",
      "arxiv_id": "2312.11819v3",
      "published": "2023-12-19",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2312.11819v3",
      "pdf_url": "https://arxiv.org/pdf/2312.11819v3.pdf"
    },
    {
      "title": "FedRLHF: A Convergence-Guaranteed Federated Framework for Privacy-Preserving and Personalized RLHF",
      "authors": [
        "Flint Xiaofeng Fan",
        "Cheston Tan",
        "Yew-Soon Ong",
        "Roger Wattenhofer",
        "Wei-Tsang Ooi"
      ],
      "summary": "In the era of increasing privacy concerns and demand for personalized experiences, traditional Reinforcement Learning with Human Feedback (RLHF) frameworks face significant challenges due to their reliance on centralized data. We introduce Federated Reinforcement Learning with Human Feedback (FedRLHF), a novel framework that decentralizes the RLHF process. FedRLHF enables collaborative policy learning across multiple clients without necessitating the sharing of raw data or human feedback, thereby ensuring robust privacy preservation. Leveraging federated reinforcement learning, each client integrates human feedback locally into their reward functions and updates their policies through personalized RLHF processes. We establish rigorous theoretical foundations for FedRLHF, providing convergence guarantees, and deriving sample complexity bounds that scale efficiently with the number of clients. Empirical evaluations on the MovieLens and IMDb datasets demonstrate that FedRLHF not only preserves user privacy but also achieves performance on par with centralized RLHF, while enhancing personalization across diverse client environments.",
      "arxiv_id": "2412.15538v2",
      "published": "2024-12-20",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CR"
      ],
      "url": "https://arxiv.org/abs/2412.15538v2",
      "pdf_url": "https://arxiv.org/pdf/2412.15538v2.pdf"
    },
    {
      "title": "Time-Myopic Go-Explore: Learning A State Representation for the Go-Explore Paradigm",
      "authors": [
        "Marc H\u00f6ftmann",
        "Jan Robine",
        "Stefan Harmeling"
      ],
      "summary": "Very large state spaces with a sparse reward signal are difficult to explore. The lack of a sophisticated guidance results in a poor performance for numerous reinforcement learning algorithms. In these cases, the commonly used random exploration is often not helpful. The literature shows that this kind of environments require enormous efforts to systematically explore large chunks of the state space. Learned state representations can help here to improve the search by providing semantic context and build a structure on top of the raw observations. In this work we introduce a novel time-myopic state representation that clusters temporal close states together while providing a time prediction capability between them. By adapting this model to the Go-Explore paradigm (Ecoffet et al., 2021b), we demonstrate the first learned state representation that reliably estimates novelty instead of using the hand-crafted representation heuristic. Our method shows an improved solution for the detachment problem which still remains an issue at the Go-Explore Exploration Phase. We provide evidence that our proposed method covers the entire state space with respect to all possible time trajectories without causing disadvantageous conflict-overlaps in the cell archive. Analogous to native Go-Explore, our approach is evaluated on the hard exploration environments MontezumaRevenge, Gravitar and Frostbite (Atari) in order to validate its capabilities on difficult tasks. Our experiments show that time-myopic Go-Explore is an effective alternative for the domain-engineered heuristic while also being more general. The source code of the method is available on GitHub.",
      "arxiv_id": "2301.05635v1",
      "published": "2023-01-13",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2301.05635v1",
      "pdf_url": "https://arxiv.org/pdf/2301.05635v1.pdf"
    },
    {
      "title": "G-Core: A Simple, Scalable and Balanced RLHF Trainer",
      "authors": [
        "Junyu Wu",
        "Weiming Chang",
        "Xiaotao Liu",
        "Guanyou He",
        "Haoqiang Hong",
        "Boqi Liu",
        "Hongtao Tian",
        "Tao Yang",
        "Yunsheng Shi",
        "Feng Lin",
        "Ting Yao"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has become an increasingly popular paradigm for training large language models (LLMs) and diffusion models. While existing RLHF training systems have enabled significant progress, they often face challenges in scaling to multi-modal and diffusion workflows and adapting to dynamic workloads. In particular, current approaches may encounter limitations in controller scalability, flexible resource placement, and efficient orchestration when handling complex RLHF pipelines, especially in scenarios involving dynamic sampling or generative reward modeling. In this paper, we present \\textbf{G-Core}, a simple, scalable, and balanced RLHF training framework designed to address these challenges. G-Core introduces a parallel controller programming model, enabling flexible and efficient orchestration of complex RLHF workflows without the bottlenecks of a single centralized controller. Furthermore, we propose a dynamic placement schema that adaptively partitions resources and schedules workloads, significantly reducing hardware idle time and improving utilization, even under highly variable training conditions. G-Core has successfully trained models that support WeChat product features serving a large-scale user base, demonstrating its effectiveness and robustness in real-world scenarios. Our results show that G-Core advances the state of the art in RLHF training, providing a solid foundation for future research and deployment of large-scale, human-aligned models.",
      "arxiv_id": "2507.22789v2",
      "published": "2025-07-30",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2507.22789v2",
      "pdf_url": "https://arxiv.org/pdf/2507.22789v2.pdf"
    },
    {
      "title": "Parameter Efficient Reinforcement Learning from Human Feedback",
      "authors": [
        "Hakim Sidahmed",
        "Samrat Phatale",
        "Alex Hutcheson",
        "Zhuonan Lin",
        "Zhang Chen",
        "Zac Yu",
        "Jarvis Jin",
        "Simral Chaudhary",
        "Roman Komarytsia",
        "Christiane Ahlheim",
        "Yonghao Zhu",
        "Bowen Li",
        "Saravanan Ganesh",
        "Bill Byrne",
        "Jessica Hoffmann",
        "Hassan Mansoor",
        "Wei Li",
        "Abhinav Rastogi",
        "Lucas Dixon"
      ],
      "summary": "While Reinforcement Learning from Human Feedback (RLHF) effectively aligns pretrained Large Language and Vision-Language Models (LLMs, and VLMs) with human preferences, its computational cost and complexity hamper its wider adoption. To alleviate some of the computational burden of fine-tuning, parameter efficient methods, like LoRA were introduced. In this work, we empirically evaluate the setup of Parameter Efficient Reinforcement Learning from Human Feedback (PE-RLHF) that leverages LoRA fine-tuning for Reward Modeling, and Reinforcement Learning. We benchmark the PE-RLHF setup on six diverse datasets spanning summarization, harmless/helpful response generation, UI automation, and visual question answering in terms of effectiveness of the trained models, and the training resources required. Our findings show, for the first time, that PE-RLHF achieves comparable performance to RLHF, while significantly reducing training time (up to 90% faster for reward models, and 30% faster for RL), and memory footprint (up to 50% reduction for reward models, and 27% for RL). We provide comprehensive ablations across LoRA ranks, and model sizes for both reward modeling and reinforcement learning. By mitigating the computational burden associated with RLHF, we push for a broader adoption of PE-RLHF as an alignment technique for LLMs and VLMs.",
      "arxiv_id": "2403.10704v2",
      "published": "2024-03-15",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2403.10704v2",
      "pdf_url": "https://arxiv.org/pdf/2403.10704v2.pdf"
    },
    {
      "title": "Does RLHF Scale? Exploring the Impacts From Data, Model, and Method",
      "authors": [
        "Zhenyu Hou",
        "Pengfan Du",
        "Yilin Niu",
        "Zhengxiao Du",
        "Aohan Zeng",
        "Xiao Liu",
        "Minlie Huang",
        "Hongning Wang",
        "Jie Tang",
        "Yuxiao Dong"
      ],
      "summary": "This study explores the scaling properties of Reinforcement Learning from Human Feedback (RLHF) in Large Language Models (LLMs). Although RLHF is considered an important step in post-training of LLMs, its scaling potential is still largely unknown. We systematically analyze key components in the RLHF framework--model size, data composition, and inference budget--and their impacts on performance. Our findings show that increasing data diversity and volume improves reward model performance, helping process-supervision models scale better. For policy training, more response samples per prompt boost performance initially but quickly plateau. And larger reward models offer modest gains in policy training. In addition, larger policy models benefit less from RLHF with a fixed reward model. Overall, RLHF scales less efficiently than pretraining, with diminishing returns from additional computational resources. Based on these observations, we propose strategies to optimize RLHF performance within computational limits.",
      "arxiv_id": "2412.06000v1",
      "published": "2024-12-08",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2412.06000v1",
      "pdf_url": "https://arxiv.org/pdf/2412.06000v1.pdf"
    },
    {
      "title": "Principled Reinforcement Learning with Human Feedback from Pairwise or $K$-wise Comparisons",
      "authors": [
        "Banghua Zhu",
        "Jiantao Jiao",
        "Michael I. Jordan"
      ],
      "summary": "We provide a theoretical framework for Reinforcement Learning with Human Feedback (RLHF). Our analysis shows that when the true reward function is linear, the widely used maximum likelihood estimator (MLE) converges under both the Bradley-Terry-Luce (BTL) model and the Plackett-Luce (PL) model. However, we show that when training a policy based on the learned reward model, MLE fails while a pessimistic MLE provides policies with improved performance under certain coverage assumptions. Additionally, we demonstrate that under the PL model, the true MLE and an alternative MLE that splits the $K$-wise comparison into pairwise comparisons both converge. Moreover, the true MLE is asymptotically more efficient. Our results validate the empirical success of existing RLHF algorithms in InstructGPT and provide new insights for algorithm design. Furthermore, our results unify the problem of RLHF and max-entropy Inverse Reinforcement Learning (IRL), and provide the first sample complexity bound for max-entropy IRL.",
      "arxiv_id": "2301.11270v5",
      "published": "2023-01-26",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.HC",
        "math.ST",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2301.11270v5",
      "pdf_url": "https://arxiv.org/pdf/2301.11270v5.pdf"
    },
    {
      "title": "Code as Reward: Empowering Reinforcement Learning with VLMs",
      "authors": [
        "David Venuto",
        "Sami Nur Islam",
        "Martin Klissarov",
        "Doina Precup",
        "Sherry Yang",
        "Ankit Anand"
      ],
      "summary": "Pre-trained Vision-Language Models (VLMs) are able to understand visual concepts, describe and decompose complex tasks into sub-tasks, and provide feedback on task completion. In this paper, we aim to leverage these capabilities to support the training of reinforcement learning (RL) agents. In principle, VLMs are well suited for this purpose, as they can naturally analyze image-based observations and provide feedback (reward) on learning progress. However, inference in VLMs is computationally expensive, so querying them frequently to compute rewards would significantly slowdown the training of an RL agent. To address this challenge, we propose a framework named Code as Reward (VLM-CaR). VLM-CaR produces dense reward functions from VLMs through code generation, thereby significantly reducing the computational burden of querying the VLM directly. We show that the dense rewards generated through our approach are very accurate across a diverse set of discrete and continuous environments, and can be more effective in training RL policies than the original sparse environment rewards.",
      "arxiv_id": "2402.04764v1",
      "published": "2024-02-07",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2402.04764v1",
      "pdf_url": "https://arxiv.org/pdf/2402.04764v1.pdf"
    },
    {
      "title": "Reinforcement Learning with Generative Models for Compact Support Sets",
      "authors": [
        "Nico Schiavone",
        "Xingyu Li"
      ],
      "summary": "Foundation models contain a wealth of information from their vast number of training samples. However, most prior arts fail to extract this information in a precise and efficient way for small sample sizes. In this work, we propose a framework utilizing reinforcement learning as a control for foundation models, allowing for the granular generation of small, focused synthetic support sets to augment the performance of neural network models on real data classification tasks. We first allow a reinforcement learning agent access to a novel context based dictionary; the agent then uses this dictionary with a novel prompt structure to form and optimize prompts as inputs to generative models, receiving feedback based on a reward function combining the change in validation accuracy and entropy. A support set is formed this way over several exploration steps. Our framework produced excellent results, increasing classification accuracy by significant margins for no additional labelling or data cost.",
      "arxiv_id": "2404.16300v1",
      "published": "2024-04-25",
      "categories": [
        "cs.LG",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2404.16300v1",
      "pdf_url": "https://arxiv.org/pdf/2404.16300v1.pdf"
    },
    {
      "title": "Dual-Agent Reinforcement Learning for Automated Feature Generation",
      "authors": [
        "Wanfu Gao",
        "Zengyao Man",
        "Hanlin Pan",
        "Kunpeng Liu"
      ],
      "summary": "Feature generation involves creating new features from raw data to capture complex relationships among the original features, improving model robustness and machine learning performance. Current methods using reinforcement learning for feature generation have made feature exploration more flexible and efficient. However, several challenges remain: first, during feature expansion, a large number of redundant features are generated. When removing them, current methods only retain the best features each round, neglecting those that perform poorly initially but could improve later. Second, the state representation used by current methods fails to fully capture complex feature relationships. Third, there are significant differences between discrete and continuous features in tabular data, requiring different operations for each type. To address these challenges, we propose a novel dual-agent reinforcement learning method for feature generation. Two agents are designed: the first generates new features, and the second determines whether they should be preserved. A self-attention mechanism enhances state representation, and diverse operations distinguish interactions between discrete and continuous features. The experimental results on multiple datasets demonstrate that the proposed method is effective. The code is available at https://github.com/extess0/DARL.",
      "arxiv_id": "2505.12628v1",
      "published": "2025-05-19",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.12628v1",
      "pdf_url": "https://arxiv.org/pdf/2505.12628v1.pdf"
    },
    {
      "title": "Reinforcement Learning-Guided Chain-of-Draft for Token-Efficient Code Generation",
      "authors": [
        "Xunzhu Tang",
        "Iyiola Emmanuel Olatunji",
        "Tiezhu Sun",
        "Jacques Klein",
        "Tegawende F. Bissyande"
      ],
      "summary": "LLMs demonstrate surface-level fluency in code generation but struggle with structured reasoning tasks requiring correctness and semantic alignment. While Chain-of-Thought (CoT) prompting enhances reasoning through intermediate steps, it suffers from verbosity and inefficiency. Chain-of-Draft (CoD) prompting offers more concise reasoning, but the stochastic nature of LLMs produces varying solution quality, making optimal selection challenging. We propose \\multicod, a reinforcement learning framework that learns to select the most promising candidate from CoD-generated solutions. Our approach uses strategy-guided prompting to encourage diverse reasoning styles and models solution selection as a contextual bandit problem. The framework optimizes interpretable features including code complexity, reasoning structure, and strategic metadata through a reward function balancing correctness, efficiency, and clarity. Experiments on MBPP, BigCodeBench, SWE-bench Verified, and Defects4J show \\multicod~outperforms and in some cases, on par with standard prompting, CoT, and CoD baselines while achieving cost and token efficiency from the user's perspective through a multi-candidate design that charges only for the selected output, reducing user billing by over 50\\% and improving LLM response quality, making \\multicod~more sustainable and scalable for real-world deployment. Our code is available: https://anonymous.4open.science/r/MultiCoD.",
      "arxiv_id": "2509.25243v1",
      "published": "2025-09-26",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2509.25243v1",
      "pdf_url": "https://arxiv.org/pdf/2509.25243v1.pdf"
    },
    {
      "title": "Greedy Sampling Is Provably Efficient for RLHF",
      "authors": [
        "Di Wu",
        "Chengshuai Shi",
        "Jing Yang",
        "Cong Shen"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a key technique for post-training large language models. Despite its empirical success, the theoretical understanding of RLHF is still limited, as learning the KL-regularized target with only preference feedback poses additional challenges compared with canonical RL. Existing works mostly study the reward-based Bradley-Terry (BT) preference model, and extend classical designs utilizing optimism or pessimism. This work, instead, considers the general preference model (whose practical relevance has been observed recently) and obtains performance guarantees with major, order-wise improvements over existing ones. Surprisingly, these results are derived from algorithms that directly use the empirical estimates (i.e., greedy sampling), as opposed to constructing optimistic or pessimistic estimates in previous works. This insight has a deep root in the unique structural property of the optimal policy class under the KL-regularized target, and we further specialize it to the BT model, highlighting the surprising sufficiency of greedy sampling in RLHF.",
      "arxiv_id": "2510.24700v1",
      "published": "2025-10-28",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.IT",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2510.24700v1",
      "pdf_url": "https://arxiv.org/pdf/2510.24700v1.pdf"
    },
    {
      "title": "CRScore++: Reinforcement Learning with Verifiable Tool and AI Feedback for Code Review",
      "authors": [
        "Manav Nitin Kapadnis",
        "Atharva Naik",
        "Carolyn Rose"
      ],
      "summary": "Reinforcement learning (RL) to improve code review comment generation requires handling unstructured outputs, making reinforcement learning (RL) feedback challenging. The two main RL approaches, namely RL with Verifiable Feedback (RLVR) and RL with AI Feedback (RLAIF), offer trade-offs: RLVR provides reliable feedback for structured tasks like code generation, while RLAIF works for unstructured outputs but is subjective. We bridge this gap with CRScore++, an RL framework that leverages both LLM-based subjective feedback and verifiable signals for training. Extending CRScore, a code review evaluation metric integrating LLMs with verifiers like linters and code smell detectors, CRScore++ transforms these signals into training rewards. We show that CRScore++ improves a weaker student model through a combination of supervised fine-tuning and RL critique from a stronger teacher model, thus enabling generalization to novel programming languages.",
      "arxiv_id": "2506.00296v1",
      "published": "2025-05-30",
      "categories": [
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2506.00296v1",
      "pdf_url": "https://arxiv.org/pdf/2506.00296v1.pdf"
    },
    {
      "title": "RLHS: Mitigating Misalignment in RLHF with Hindsight Simulation",
      "authors": [
        "Kaiqu Liang",
        "Haimin Hu",
        "Ryan Liu",
        "Thomas L. Griffiths",
        "Jaime Fern\u00e1ndez Fisac"
      ],
      "summary": "While Reinforcement Learning from Human Feedback (RLHF) has shown promise in aligning generative AI, we present empirical evidence that it can also cause severe, systematic misalignment. We hypothesize that this stems from evaluator feedback depending on downstream outcome predictions (foresight) that can be influenced by the AI's output, inducing Goodhart's law dynamics. We present a theoretical analysis showing that conditioning evaluator feedback on downstream observations (hindsight) inhibits this effect by decoupling the alignment signal from potentially compromised predictions--crucially, the result holds even if the observed outcomes are sampled from the AI's own world model. Building on this insight, we introduce Reinforcement Learning from Hindsight Simulation (RLHS), which presents plausible simulated outcomes to evaluators before eliciting feedback. We validate RLHS across three consultancy settings--marketplace interactions, restaurant recommendations, and online course advising--using both online (PPO) and offline (DPO) fine-tuning methods, and show that it substantially improves alignment over RLHF in experiments and human evaluations. We perform post-hoc benchmark evaluations on TruthfulQA, HaluEval, and TrustLLM, finding that even after single-task fine-tuning, RLHF misalignment persists, whereas RLHS consistently outperforms baselines and demonstrates robust alignment generalization. The project webpage and code are available at https://rl-hindsight.github.io.",
      "arxiv_id": "2501.08617v3",
      "published": "2025-01-15",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.08617v3",
      "pdf_url": "https://arxiv.org/pdf/2501.08617v3.pdf"
    },
    {
      "title": "Reinforcement Learning from Human Feedback",
      "authors": [
        "Nathan Lambert"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has become an important technical and storytelling tool to deploy the latest machine learning systems. In this book, we hope to give a gentle introduction to the core methods for people with some level of quantitative background. The book starts with the origins of RLHF -- both in recent literature and in a convergence of disparate fields of science in economics, philosophy, and optimal control. We then set the stage with definitions, problem formulation, data collection, and other common math used in the literature. The core of the book details every optimization stage in using RLHF, from starting with instruction tuning to training a reward model and finally all of rejection sampling, reinforcement learning, and direct alignment algorithms. The book concludes with advanced topics -- understudied research questions in synthetic data and evaluation -- and open questions for the field.",
      "arxiv_id": "2504.12501v3",
      "published": "2025-04-16",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.12501v3",
      "pdf_url": "https://arxiv.org/pdf/2504.12501v3.pdf"
    },
    {
      "title": "SALSA: Soup-based Alignment Learning for Stronger Adaptation in RLHF",
      "authors": [
        "Atoosa Chegini",
        "Hamid Kazemi",
        "Iman Mirzadeh",
        "Dong Yin",
        "Maxwell Horton",
        "Moin Nabi",
        "Mehrdad Farajtabar",
        "Keivan Alizadeh"
      ],
      "summary": "In Large Language Model (LLM) development, Reinforcement Learning from Human Feedback (RLHF) is crucial for aligning models with human values and preferences. RLHF traditionally relies on the Kullback-Leibler (KL) divergence between the current policy and a frozen initial policy as a reference, which is added as a penalty in policy optimization algorithms like Proximal Policy Optimization (PPO). While this constraint prevents models from deviating too far from the initial checkpoint, it limits exploration of the reward landscape, reducing the model's ability to discover higher-quality solutions. As a result, policy optimization is often trapped in a narrow region of the parameter space, leading to suboptimal alignment and performance. This paper presents SALSA (Soup-based Alignment Learning for Stronger Adaptation), a novel approach designed to overcome these limitations by creating a more flexible and better located reference model through weight-space averaging of two independent supervised fine-tuned (SFT) models. This model soup allows for larger deviation in KL divergence and exploring a promising region of the solution space without sacrificing stability. By leveraging this more robust reference model, SALSA fosters better exploration, achieving higher rewards and improving model robustness, out-of-distribution generalization, and performance. We validate the effectiveness of SALSA through extensive experiments on popular open models (Llama2-7B, Mistral-7B, and Gemma-2B) across various benchmarks (MT-Bench, Arena-Hard, UltraFeedback), where it consistently surpasses PPO by fostering deeper exploration and achieving superior alignment in LLMs.",
      "arxiv_id": "2411.01798v1",
      "published": "2024-11-04",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2411.01798v1",
      "pdf_url": "https://arxiv.org/pdf/2411.01798v1.pdf"
    },
    {
      "title": "Provable Multi-Party Reinforcement Learning with Diverse Human Feedback",
      "authors": [
        "Huiying Zhong",
        "Zhun Deng",
        "Weijie J. Su",
        "Zhiwei Steven Wu",
        "Linjun Zhang"
      ],
      "summary": "Reinforcement learning with human feedback (RLHF) is an emerging paradigm to align models with human preferences. Typically, RLHF aggregates preferences from multiple individuals who have diverse viewpoints that may conflict with each other. Our work \\textit{initiates} the theoretical study of multi-party RLHF that explicitly models the diverse preferences of multiple individuals. We show how traditional RLHF approaches can fail since learning a single reward function cannot capture and balance the preferences of multiple individuals. To overcome such limitations, we incorporate meta-learning to learn multiple preferences and adopt different social welfare functions to aggregate the preferences across multiple parties. We focus on the offline learning setting and establish sample complexity bounds, along with efficiency and fairness guarantees, for optimizing diverse social welfare functions such as Nash, Utilitarian, and Leximin welfare functions. Our results show a separation between the sample complexities of multi-party RLHF and traditional single-party RLHF. Furthermore, we consider a reward-free setting, where each individual's preference is no longer consistent with a reward model, and give pessimistic variants of the von Neumann Winner based on offline preference data. Taken together, our work showcases the advantage of multi-party RLHF but also highlights its more demanding statistical complexity.",
      "arxiv_id": "2403.05006v1",
      "published": "2024-03-08",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ME",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2403.05006v1",
      "pdf_url": "https://arxiv.org/pdf/2403.05006v1.pdf"
    },
    {
      "title": "Low-Rank Contextual Reinforcement Learning from Heterogeneous Human Feedback",
      "authors": [
        "Seong Jin Lee",
        "Will Wei Sun",
        "Yufeng Liu"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has become a cornerstone for aligning large language models with human preferences. However, the heterogeneity of human feedback, driven by diverse individual contexts and preferences, poses significant challenges for reward learning. To address this, we propose a Low-rank Contextual RLHF (LoCo-RLHF) framework that integrates contextual information to better model heterogeneous feedback while maintaining computational efficiency. Our approach builds on a contextual preference model, leveraging the intrinsic low-rank structure of the interaction between user contexts and query-answer pairs to mitigate the high dimensionality of feature representations. Furthermore, we address the challenge of distributional shifts in feedback through our Pessimism in Reduced Subspace (PRS) policy, inspired by pessimistic offline reinforcement learning techniques. We theoretically demonstrate that our policy achieves a tighter sub-optimality gap compared to existing methods. Extensive experiments validate the effectiveness of LoCo-RLHF, showcasing its superior performance in personalized RLHF settings and its robustness to distribution shifts.",
      "arxiv_id": "2412.19436v1",
      "published": "2024-12-27",
      "categories": [
        "stat.ML",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2412.19436v1",
      "pdf_url": "https://arxiv.org/pdf/2412.19436v1.pdf"
    },
    {
      "title": "Dr. Boot: Bootstrapping Program Synthesis Language Models to Perform Repairing",
      "authors": [
        "Noah van der Vleuten"
      ],
      "summary": "Language models for program synthesis are usually trained and evaluated on programming competition datasets (MBPP, APPS). However, these datasets are limited in size and quality, while these language models are extremely data hungry. Additionally, the language models have a misaligned program synthesis process compared to humans. While humans iteratively develop code with the help of a compiler, most program synthesis models currently produce code in one go. To solve these issues, we introduce a bootstrapping algorithm for program synthesis, that supports teaching models how to repair. We show that bootstrapping consistently outperforms regular fine-tuning. Compared to other work, our bootstrapped model performs on par with fine-tuned models that are 68\\% larger. Notably, bootstrapping with repairing also improves non-repairing performance compared to regular bootstrapping during inference. However, on our models, repairing during inference is likely inferior to simply sampling the same number of solutions. Furthermore, we find that there are issues with the example test cases in the training portion of the APPS dataset that are valuable to the community, as many repairing and reinforcement learning methods rely on them.",
      "arxiv_id": "2507.15889v1",
      "published": "2025-07-20",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2507.15889v1",
      "pdf_url": "https://arxiv.org/pdf/2507.15889v1.pdf"
    },
    {
      "title": "Delving into RL for Image Generation with CoT: A Study on DPO vs. GRPO",
      "authors": [
        "Chengzhuo Tong",
        "Ziyu Guo",
        "Renrui Zhang",
        "Wenyu Shan",
        "Xinyu Wei",
        "Zhenghao Xing",
        "Hongsheng Li",
        "Pheng-Ann Heng"
      ],
      "summary": "Recent advancements underscore the significant role of Reinforcement Learning (RL) in enhancing the Chain-of-Thought (CoT) reasoning capabilities of large language models (LLMs). Two prominent RL algorithms, Direct Preference Optimization (DPO) and Group Relative Policy Optimization (GRPO), are central to these developments, showcasing different pros and cons. Autoregressive image generation, also interpretable as a sequential CoT reasoning process, presents unique challenges distinct from LLM-based CoT reasoning. These encompass ensuring text-image consistency, improving image aesthetic quality, and designing sophisticated reward models, rather than relying on simpler rule-based rewards. While recent efforts have extended RL to this domain, these explorations typically lack an in-depth analysis of the domain-specific challenges and the characteristics of different RL strategies. To bridge this gap, we provide the first comprehensive investigation of the GRPO and DPO algorithms in autoregressive image generation, evaluating their in-domain performance and out-of-domain generalization, while scrutinizing the impact of different reward models on their respective capabilities. Our findings reveal that GRPO and DPO exhibit distinct advantages, and crucially, that reward models possessing stronger intrinsic generalization capabilities potentially enhance the generalization potential of the applied RL algorithms. Furthermore, we systematically explore three prevalent scaling strategies to enhance both their in-domain and out-of-domain proficiency, deriving unique insights into efficiently scaling performance for each paradigm. We hope our study paves a new path for inspiring future work on developing more effective RL algorithms to achieve robust CoT reasoning in the realm of autoregressive image generation. Code is released at https://github.com/ZiyuGuo99/Image-Generation-CoT",
      "arxiv_id": "2505.17017v2",
      "published": "2025-05-22",
      "categories": [
        "cs.CV",
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.17017v2",
      "pdf_url": "https://arxiv.org/pdf/2505.17017v2.pdf"
    },
    {
      "title": "A Shared Low-Rank Adaptation Approach to Personalized RLHF",
      "authors": [
        "Renpu Liu",
        "Peng Wang",
        "Donghao Li",
        "Cong Shen",
        "Jing Yang"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a pivotal technique for aligning artificial intelligence systems with human values, achieving remarkable success in fine-tuning large language models. However, existing RLHF frameworks often assume that human preferences are relatively homogeneous and can be captured by a single, unified reward model. This assumption overlooks the inherent diversity and heterogeneity across individuals, limiting the adaptability of RLHF to personalized scenarios and risking misalignments that can diminish user satisfaction and trust in AI systems. In this paper, we address these challenges by introducing Low-Rank Adaptation (LoRA) into the personalized RLHF framework. We apply LoRA in the the aggregated parameter space of all personalized reward functions, thereby enabling efficient learning of personalized reward models from potentially limited local datasets. Our approach exploits potential shared structures among the local ground-truth reward models while allowing for individual adaptation, without relying on restrictive assumptions about shared representations as in prior works. We further establish sample complexity guarantees for our method. Theoretical analysis demonstrates the effectiveness of the proposed approach in capturing both shared and individual-specific structures within heterogeneous human preferences, addressing the dual challenge of personalization requirements and practical data constraints. Experimental results on real-world datasets corroborate the efficiency of our algorithm in the personalized RLHF setting.",
      "arxiv_id": "2503.19201v1",
      "published": "2025-03-24",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.19201v1",
      "pdf_url": "https://arxiv.org/pdf/2503.19201v1.pdf"
    },
    {
      "title": "Simplify RLHF as Reward-Weighted SFT: A Variational Method",
      "authors": [
        "Yuhao Du",
        "Zhuo Li",
        "Pengyu Cheng",
        "Zhihong Chen",
        "Yuejiao Xie",
        "Xiang Wan",
        "Anningzhe Gao"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is crucial for aligning Large Language Models (LLMs) with human values. However, RLHF has been continuously challenged by its high complexity in implementation and computation consumption. Even with recent simplifications, such as Direct Preference Optimization (DPO) and Advantage Leftover Lunch (A-LoL), the problems of over-fitting and training instability remain hindering the alignment process from the expected optimal performance. To address the existing challenges, we propose a novel simplification of RLHF from the perspective of variational inference, called $\\textbf{V}$ariational $\\textbf{A}$lignment with $\\textbf{R}$e-weighting ($\\textbf{VAR}$). More specifically, by directly minimizing the distribution gap between the learning LLM policy and the optimal solution of RLHF, we transform the alignment objective into a reward-driven re-weighted supervised fine-tuning (SFT) form, which only requires minor adjustment on the SFT loss to obtain noticeable improvement on training stability and effectiveness. On comprehensive alignment and generation benchmarks, our VAR method has numerically achieved competitive performance in LLM alignment helpfulness and harmlessness.",
      "arxiv_id": "2502.11026v2",
      "published": "2025-02-16",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2502.11026v2",
      "pdf_url": "https://arxiv.org/pdf/2502.11026v2.pdf"
    },
    {
      "title": "Thompson Sampling in Online RLHF with General Function Approximation",
      "authors": [
        "Songtao Feng",
        "Jie Fu"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has achieved great empirical success in aligning large language models (LLMs) with human preference, and it is of great importance to study the statistical efficiency of RLHF algorithms from a theoretical perspective. In this work, we consider the online RLHF setting where the preference data is revealed during the learning process and study action value function approximation. We design a model-free posterior sampling algorithm for online RLHF inspired by Thompson sampling and provide its theoretical guarantee. Specifically, we adopt Bellman eluder (BE) dimension as the complexity measure of the function class and establish $O(\\sqrt{T})$ regret bound for the proposed algorithm with other multiplicative factor depending on the horizon, BE dimension and the $log$-bracketing number of the function class. Further, in the analysis, we first establish the concentration-type inequality of the squared Bellman error bound based on the maximum likelihood estimator (MLE) generalization bound, which plays the crucial rules in obtaining the eluder-type regret bound and may be of independent interest.",
      "arxiv_id": "2505.23927v1",
      "published": "2025-05-29",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.23927v1",
      "pdf_url": "https://arxiv.org/pdf/2505.23927v1.pdf"
    },
    {
      "title": "Avoiding $\\mathbf{exp(R_{max})}$ scaling in RLHF through Preference-based Exploration",
      "authors": [
        "Mingyu Chen",
        "Yiding Chen",
        "Wen Sun",
        "Xuezhou Zhang"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a pivotal technique for large language model (LLM) alignment. This paper studies the setting of online RLHF and focus on improving sample efficiency. All existing algorithms in online RLHF, whether doing passive exploration or active exploration, suffer from a sample complexity that scales exponentially with the scale of the reward function. This fundamental limitation hinders their effectiveness in scenarios with heavily skewed preferences, e.g. questions with a unique correct solution. To address this, we introduce Self-Exploring Preference-Incentive Online Preference Optimization (SE-POPO), an online RLHF algorithm that for the first time achieves a sample complexity that scales polynomially with the reward scale, answering an open problem raised by Xie et al. (2024).. Theoretically, we demonstrate that the sample complexity of SE-POPO dominates that of existing exploration algorithms. Empirically, our systematic evaluation confirms that SE-POPO is more sample-efficient than both exploratory and non-exploratory baselines, in two primary application scenarios of RLHF as well as on public benchmarks, marking a significant step forward in RLHF algorithm design. The code is available at https://github.com/MYC000801/SE-POPO.",
      "arxiv_id": "2502.00666v3",
      "published": "2025-02-02",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2502.00666v3",
      "pdf_url": "https://arxiv.org/pdf/2502.00666v3.pdf"
    },
    {
      "title": "Provably Efficient Online RLHF with One-Pass Reward Modeling",
      "authors": [
        "Long-Fei Li",
        "Yu-Yang Qian",
        "Peng Zhao",
        "Zhi-Hua Zhou"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has shown remarkable success in aligning Large Language Models (LLMs) with human preferences. Traditional RLHF methods rely on a fixed dataset, which often suffers from limited coverage. To this end, online RLHF has emerged as a promising direction, enabling iterative data collection and refinement. Despite its potential, this paradigm faces a key bottleneck: the requirement to continuously integrate new data into the dataset and re-optimize the model from scratch at each iteration, resulting in computational and storage costs that grow linearly with the number of iterations. In this work, we address this challenge by proposing a one-pass reward modeling method that eliminates the need to store historical data and achieves constant-time updates per iteration. Specifically, we first formalize RLHF as a contextual preference bandit and develop a new algorithm based on online mirror descent with a tailored local norm, replacing the standard maximum likelihood estimation for reward modeling. We then apply it to various online RLHF settings, including passive data collection, active data collection, and deployment-time adaptation. We provide theoretical guarantees showing that our method enhances both statistical and computational efficiency. Finally, we design practical algorithms for LLMs and conduct experiments with the Llama-3-8B-Instruct and Qwen2.5-7B-Instruct models on Ultrafeedback and Mixture2 datasets, validating the effectiveness of our approach.",
      "arxiv_id": "2502.07193v3",
      "published": "2025-02-11",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2502.07193v3",
      "pdf_url": "https://arxiv.org/pdf/2502.07193v3.pdf"
    },
    {
      "title": "RLHF and IIA: Perverse Incentives",
      "authors": [
        "Wanqiao Xu",
        "Shi Dong",
        "Xiuyuan Lu",
        "Grace Lam",
        "Zheng Wen",
        "Benjamin Van Roy"
      ],
      "summary": "Existing algorithms for reinforcement learning from human feedback (RLHF) can incentivize responses at odds with preferences because they are based on models that assume independence of irrelevant alternatives (IIA). The perverse incentives induced by IIA hinder innovations on query formats and learning algorithms.",
      "arxiv_id": "2312.01057v3",
      "published": "2023-12-02",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2312.01057v3",
      "pdf_url": "https://arxiv.org/pdf/2312.01057v3.pdf"
    },
    {
      "title": "Reward-Robust RLHF in LLMs",
      "authors": [
        "Yuzi Yan",
        "Xingzhou Lou",
        "Jialian Li",
        "Yiping Zhang",
        "Jian Xie",
        "Chao Yu",
        "Yu Wang",
        "Dong Yan",
        "Yuan Shen"
      ],
      "summary": "As Large Language Models (LLMs) continue to progress toward more advanced forms of intelligence, Reinforcement Learning from Human Feedback (RLHF) is increasingly seen as a key pathway toward achieving Artificial General Intelligence (AGI). However, the reliance on reward-model-based (RM-based) alignment methods introduces significant challenges due to the inherent instability and imperfections of Reward Models (RMs), which can lead to critical issues such as reward hacking and misalignment with human intentions. In this paper, we introduce a reward-robust RLHF framework aimed at addressing these fundamental challenges, paving the way for more reliable and resilient learning in LLMs. Our approach introduces a novel optimization objective that carefully balances performance and robustness by incorporating Bayesian Reward Model Ensembles (BRME) to model the uncertainty set of reward functions. This allows the framework to integrate both nominal performance and minimum reward signals, ensuring more stable learning even with imperfect RMs. Empirical results demonstrate that our framework consistently outperforms baselines across diverse benchmarks, showing improved accuracy and long-term stability. We also provide a theoretical analysis, demonstrating that reward-robust RLHF approaches the stability of constant reward settings, which proves to be acceptable even in a stochastic-case analysis. Together, these contributions highlight the framework potential to enhance both the performance and stability of LLM alignment.",
      "arxiv_id": "2409.15360v3",
      "published": "2024-09-18",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2409.15360v3",
      "pdf_url": "https://arxiv.org/pdf/2409.15360v3.pdf"
    },
    {
      "title": "Reinforcement Learning from Statistical Feedback: the Journey from AB Testing to ANT Testing",
      "authors": [
        "Feiyang Han",
        "Yimin Wei",
        "Zhaofeng Liu",
        "Yanxing Qi"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has played a crucial role in the success of large models such as ChatGPT. RLHF is a reinforcement learning framework which combines human feedback to improve learning effectiveness and performance. However, obtaining preferences feedback manually is quite expensive in commercial applications. Some statistical commercial indicators are usually more valuable and always ignored in RLHF. There exists a gap between commercial target and model training. In our research, we will attempt to fill this gap with statistical business feedback instead of human feedback, using AB testing which is a well-established statistical method. Reinforcement Learning from Statistical Feedback (RLSF) based on AB testing is proposed. Statistical inference methods are used to obtain preferences for training the reward network, which fine-tunes the pre-trained model in reinforcement learning framework, achieving greater business value. Furthermore, we extend AB testing with double selections at a single time-point to ANT testing with multiple selections at different feedback time points. Moreover, we design numerical experiences to validate the effectiveness of our algorithm framework.",
      "arxiv_id": "2311.14766v1",
      "published": "2023-11-24",
      "categories": [
        "cs.LG",
        "math.ST",
        "stat.ME"
      ],
      "url": "https://arxiv.org/abs/2311.14766v1",
      "pdf_url": "https://arxiv.org/pdf/2311.14766v1.pdf"
    },
    {
      "title": "Proxy-RLHF: Decoupling Generation and Alignment in Large Language Model with Proxy",
      "authors": [
        "Yu Zhu",
        "Chuxiong Sun",
        "Wenfei Yang",
        "Wenqiang Wei",
        "Bo Tang",
        "Tianzhu Zhang",
        "Zhiyu Li",
        "Shifeng Zhang",
        "Feiyu Xiong",
        "Jie Hu",
        "Mingchuan yang"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is the prevailing approach to ensure Large Language Models (LLMs) align with human values. However, existing RLHF methods require a high computational cost, one main reason being that RLHF assigns both the generation and alignment tasks to the LLM simultaneously. In this paper, we introduce Proxy-RLHF, which decouples the generation and alignment processes of LLMs, achieving alignment with human values at a much lower computational cost. We start with a novel Markov Decision Process (MDP) designed for the alignment process and employ Reinforcement Learning (RL) to train a streamlined proxy model that oversees the token generation of the LLM, without altering the LLM itself. Experiments show that our method achieves a comparable level of alignment with only 1\\% of the training parameters of other methods.",
      "arxiv_id": "2403.04283v1",
      "published": "2024-03-07",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2403.04283v1",
      "pdf_url": "https://arxiv.org/pdf/2403.04283v1.pdf"
    },
    {
      "title": "OpenRLHF: An Easy-to-use, Scalable and High-performance RLHF Framework",
      "authors": [
        "Jian Hu",
        "Xibin Wu",
        "Wei Shen",
        "Jason Klein Liu",
        "Zilin Zhu",
        "Weixun Wang",
        "Songlin Jiang",
        "Haoran Wang",
        "Hao Chen",
        "Bin Chen",
        "Weikai Fang",
        " Xianyu",
        "Yu Cao",
        "Haotian Xu",
        "Yiming Liu"
      ],
      "summary": "Large Language Models (LLMs) fine-tuned via Reinforcement Learning from Human Feedback (RLHF) and Reinforcement Learning with Verifiable Rewards (RLVR) significantly improve the alignment of human-AI values, further raising the upper bound of AI capabilities, particularly in reasoning-intensive, long-context Chain-of-Thought (CoT) tasks. However, existing frameworks commonly face challenges such as inference bottlenecks and complexity barriers, which restrict their accessibility to newcomers. To bridge this gap, we introduce \\textbf{OpenRLHF}, a user-friendly, scalable, and easy-to-learn open-source RLHF framework built upon Ray, vLLM, DeepSpeed, and HuggingFace Transformers, featuring a simplified design, clear code structure, and comprehensive documentation to facilitate entry for researchers and practitioners. Experimental results show that OpenRLHF achieves superior training efficiency, with speedups ranging from 1.22x to 1.68x across different model sizes, compared to state-of-the-art frameworks. Additionally, it requires significantly fewer lines of code for implementation. OpenRLHF is publicly available at https://github.com/OpenRLHF/OpenRLHF, and has already been adopted by leading institutions to accelerate RLHF research and learning.",
      "arxiv_id": "2405.11143v6",
      "published": "2024-05-20",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.11143v6",
      "pdf_url": "https://arxiv.org/pdf/2405.11143v6.pdf"
    },
    {
      "title": "No $D_{\\text{train}}$: Model-Agnostic Counterfactual Explanations Using Reinforcement Learning",
      "authors": [
        "Xiangyu Sun",
        "Raquel Aoki",
        "Kevin H. Wilson"
      ],
      "summary": "Machine learning (ML) methods have experienced significant growth in the past decade, yet their practical application in high-impact real-world domains has been hindered by their opacity. When ML methods are responsible for making critical decisions, stakeholders often require insights into how to alter these decisions. Counterfactual explanations (CFEs) have emerged as a solution, offering interpretations of opaque ML models and providing a pathway to transition from one decision to another. However, most existing CFE methods require access to the model's training dataset, few methods can handle multivariate time-series, and none of model-agnostic CFE methods can handle multivariate time-series without training datasets. These limitations can be formidable in many scenarios. In this paper, we present NTD-CFE, a novel model-agnostic CFE method based on reinforcement learning (RL) that generates CFEs when training datasets are unavailable. NTD-CFE is suitable for both static and multivariate time-series datasets with continuous and discrete features. NTD-CFE reduces the CFE search space from a multivariate time-series domain to a lower dimensional space and addresses the problem using RL. Users have the flexibility to specify non-actionable, immutable, and preferred features, as well as causal constraints. We demonstrate the performance of NTD-CFE against four baselines on several datasets and find that, despite not having access to a training dataset, NTD-CFE finds CFEs that make significantly fewer and significantly smaller changes to the input time-series. These properties make CFEs more actionable, as the magnitude of change required to alter an outcome is vastly reduced. The code is available in the supplementary material.",
      "arxiv_id": "2405.18563v2",
      "published": "2024-05-28",
      "categories": [
        "cs.LG",
        "stat.ME"
      ],
      "url": "https://arxiv.org/abs/2405.18563v2",
      "pdf_url": "https://arxiv.org/pdf/2405.18563v2.pdf"
    },
    {
      "title": "Policy Filtration for RLHF to Mitigate Noise in Reward Models",
      "authors": [
        "Chuheng Zhang",
        "Wei Shen",
        "Li Zhao",
        "Xuyun Zhang",
        "Xiaolong Xu",
        "Wanchun Dou",
        "Jiang Bian"
      ],
      "summary": "While direct policy optimization methods exist, pioneering LLMs are fine-tuned with reinforcement learning from human feedback (RLHF) to generate better responses under the supervision of a reward model learned from preference data. One major challenge of RLHF is the inaccuracy of the intermediate reward model, especially in the tasks that requires complex reasoning for the reward model to score a response. We find that the reliability of the reward model varies across responses assigned with different rewards. This motivates us to filter the samples whose rewards may be unreliable to improve the signal-to-noise ratio during policy learning, resulting in Policy Filtration for Proximal Policy Optimization (PF-PPO). To choose a proper policy filtering strategy, we use the coefficient of determination (R2) between the rewards and actual scores on filtered samples as the metrics to help us find promising strategies since it measures how well the rewards filtered by PF-PPO indicate real performance. We provide extensive experiments to validate the effectiveness of PF-PPO in code generation and math reasoning tasks. In code generation, PF-PPO achieves the state-of-the-art performance of 7-billion-parameter models on HumanEval (+7.9%), MBPP (+0.7%), and LeetCode Contest (+10.0%) which is a more challenging benchmark created by us. In math reasoning, PF-PPO yields performance increase using different reward models and benchmarks (Ape210K and CMATH). Code is available on https://github.com/DtYXs/verl/tree/pf-ppo.",
      "arxiv_id": "2409.06957v5",
      "published": "2024-09-11",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2409.06957v5",
      "pdf_url": "https://arxiv.org/pdf/2409.06957v5.pdf"
    },
    {
      "title": "Exploration-Driven Policy Optimization in RLHF: Theoretical Insights on Efficient Data Utilization",
      "authors": [
        "Yihan Du",
        "Anna Winnicki",
        "Gal Dalal",
        "Shie Mannor",
        "R. Srikant"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has achieved impressive empirical successes while relying on a small amount of human feedback. However, there is limited theoretical justification for this phenomenon. Additionally, most recent studies focus on value-based algorithms despite the recent empirical successes of policy-based algorithms. In this work, we consider an RLHF algorithm based on policy optimization (PO-RLHF). The algorithm is based on the popular Policy Cover-Policy Gradient (PC-PG) algorithm, which assumes knowledge of the reward function. In PO-RLHF, knowledge of the reward function is not assumed, and the algorithm uses trajectory-based comparison feedback to infer the reward function. We provide performance bounds for PO-RLHF with low query complexity, which provides insight into why a small amount of human feedback may be sufficient to achieve good performance with RLHF. A key novelty is a trajectory-level elliptical potential analysis, which bounds the reward estimation error when comparison feedback (rather than numerical reward observation) is given. We provide and analyze algorithms PG-RLHF and NN-PG-RLHF for two settings: linear and neural function approximation, respectively.",
      "arxiv_id": "2402.10342v2",
      "published": "2024-02-15",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2402.10342v2",
      "pdf_url": "https://arxiv.org/pdf/2402.10342v2.pdf"
    },
    {
      "title": "An introduction to reinforcement learning for neuroscience",
      "authors": [
        "Kristopher T. Jensen"
      ],
      "summary": "Reinforcement learning (RL) has a rich history in neuroscience, from early work on dopamine as a reward prediction error signal (Schultz et al., 1997) to recent work proposing that the brain could implement a form of 'distributional reinforcement learning' popularized in machine learning (Dabney et al., 2020). There has been a close link between theoretical advances in reinforcement learning and neuroscience experiments throughout this literature, and the theories describing the experimental data have therefore become increasingly complex. Here, we provide an introduction and mathematical background to many of the methods that have been used in systems neroscience. We start with an overview of the RL problem and classical temporal difference algorithms, followed by a discussion of 'model-free', 'model-based', and intermediate RL algorithms. We then introduce deep reinforcement learning and discuss how this framework has led to new insights in neuroscience. This includes a particular focus on meta-reinforcement learning (Wang et al., 2018) and distributional RL (Dabney et al., 2020). Finally, we discuss potential shortcomings of the RL formalism for neuroscience and highlight open questions in the field. Code that implements the methods discussed and generates the figures is also provided.",
      "arxiv_id": "2311.07315v3",
      "published": "2023-11-13",
      "categories": [
        "q-bio.NC",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2311.07315v3",
      "pdf_url": "https://arxiv.org/pdf/2311.07315v3.pdf"
    },
    {
      "title": "A Survey on Explainable Deep Reinforcement Learning",
      "authors": [
        "Zelei Cheng",
        "Jiahao Yu",
        "Xinyu Xing"
      ],
      "summary": "Deep Reinforcement Learning (DRL) has achieved remarkable success in sequential decision-making tasks across diverse domains, yet its reliance on black-box neural architectures hinders interpretability, trust, and deployment in high-stakes applications. Explainable Deep Reinforcement Learning (XRL) addresses these challenges by enhancing transparency through feature-level, state-level, dataset-level, and model-level explanation techniques. This survey provides a comprehensive review of XRL methods, evaluates their qualitative and quantitative assessment frameworks, and explores their role in policy refinement, adversarial robustness, and security. Additionally, we examine the integration of reinforcement learning with Large Language Models (LLMs), particularly through Reinforcement Learning from Human Feedback (RLHF), which optimizes AI alignment with human preferences. We conclude by highlighting open research challenges and future directions to advance the development of interpretable, reliable, and accountable DRL systems.",
      "arxiv_id": "2502.06869v1",
      "published": "2025-02-08",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2502.06869v1",
      "pdf_url": "https://arxiv.org/pdf/2502.06869v1.pdf"
    },
    {
      "title": "Theoretical Tensions in RLHF: Reconciling Empirical Success with Inconsistencies in Social Choice Theory",
      "authors": [
        "Jiancong Xiao",
        "Zhekun Shi",
        "Kaizhao Liu",
        "Qi Long",
        "Weijie J. Su"
      ],
      "summary": "Despite its empirical success, Reinforcement Learning from Human Feedback (RLHF) has been shown to violate almost all the fundamental axioms in social choice theory -- such as majority consistency, pairwise majority consistency, and Condorcet consistency. This raises a foundational question: why does RLHF perform so well in practice if it fails these seemingly essential properties? In this paper, we resolve this paradox by showing that under mild and empirically plausible assumptions on the preference profile, RLHF does satisfy pairwise majority and Condorcet consistency. These assumptions are frequently satisfied in real-world alignment tasks, offering a theoretical explanation for RLHF's strong practical performance. Furthermore, we show that a slight modification to the reward modeling objective can ensure pairwise majority or Condorcet consistency even under general preference profiles, thereby improving the alignment process. Finally, we go beyond classical axioms in economic and social choice theory and introduce new alignment criteria -- preference matching, preference equivalence, and group preference matching -- that better reflect the goal of learning distributions over responses. We show that while RLHF satisfies the first two properties, it fails to satisfy the third. We conclude by discussing how future alignment methods may be designed to satisfy all three.",
      "arxiv_id": "2506.12350v1",
      "published": "2025-06-14",
      "categories": [
        "stat.ML",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2506.12350v1",
      "pdf_url": "https://arxiv.org/pdf/2506.12350v1.pdf"
    },
    {
      "title": "A Long Way to Go: Investigating Length Correlations in RLHF",
      "authors": [
        "Prasann Singhal",
        "Tanya Goyal",
        "Jiacheng Xu",
        "Greg Durrett"
      ],
      "summary": "Great success has been reported using Reinforcement Learning from Human Feedback (RLHF) to align large language models, with open preference datasets enabling wider experimentation, particularly for \"helpfulness\" in tasks like dialogue and web question answering. Alongside these improvements, however, RLHF also often drives models to produce longer outputs. This paper demonstrates, on three diverse settings, that optimizing for response length is, much more than previously thought, a significant factor behind RLHF. Studying the strategies RL optimization uses to maximize reward, we find improvements in reward to largely be driven by increasing response length, instead of other features. Indeed, we find that even a purely length-based reward reproduces most downstream RLHF improvements over supervised fine-tuned models. Testing a comprehensive set of length-countering interventions, we identify the dominant source of these biases to be reward models, which, by studying training dynamics, we find are non-robust and easily influenced by length biases in preference data.",
      "arxiv_id": "2310.03716v2",
      "published": "2023-10-05",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2310.03716v2",
      "pdf_url": "https://arxiv.org/pdf/2310.03716v2.pdf"
    },
    {
      "title": "ChipSeek-R1: Generating Human-Surpassing RTL with LLM via Hierarchical Reward-Driven Reinforcement Learning",
      "authors": [
        "Zhirong Chen",
        "Kaiyan Chang",
        "Zhuolin Li",
        "Xinyang He",
        "Chujie Chen",
        "Cangyuan Li",
        "Mengdi Wang",
        "Haobo Xu",
        "Yinhe Han",
        "Ying Wang"
      ],
      "summary": "Large Language Models (LLMs) show significant potential for automating Register-Transfer Level (RTL) code generation. However, current approaches face a critical challenge: they can not simultaneously optimize for functional correctness and hardware quality (Power, Performance, Area - PPA). Methods based on supervised fine-tuning often generate functionally correct but PPA-suboptimal code, lacking mechanisms to learn optimization principles. In contrast, post-processing techniques that attempt to improve PPA metrics after generation are often inefficient because they operate externally without updating the LLM's parameters, thus failing to enhance the model's intrinsic design capabilities.   To bridge this gap, we introduce ChipSeek-R1, a hierarchical reward-driven reinforcement learning framework to train LLMs to generate RTL code that achieves both functional correctness and optimized PPA metrics. ChipSeek-R1 employs a hierarchical reward system, which incorporates direct feedback on syntax, functional correctness (from simulators) and PPA metrics (from synthesis tools) during reinforcement learning. This enables the model to learn complex hardware design trade-offs via trial-and-error, generating RTL code that is both functionally correct and PPA-optimized. Evaluating ChipSeek-R1 on standard benchmarks (VerilogEval, RTLLM), we achieve state-of-the-art results in functional correctness. Notably, on the RTLLM benchmark, ChipSeek-R1 generated 27 RTL designs surpassing the PPA metrics of the original human-written code. Our findings demonstrate the effectiveness of integrating toolchain feedback into LLM training and highlight the potential for reinforcement learning to enable automated generation of human-surpassing RTL code. We open-source our code in anonymous github.",
      "arxiv_id": "2507.04736v1",
      "published": "2025-07-07",
      "categories": [
        "cs.AI",
        "cs.AR",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2507.04736v1",
      "pdf_url": "https://arxiv.org/pdf/2507.04736v1.pdf"
    },
    {
      "title": "RLTF: Reinforcement Learning from Unit Test Feedback",
      "authors": [
        "Jiate Liu",
        "Yiqin Zhu",
        "Kaiwen Xiao",
        "Qiang Fu",
        "Xiao Han",
        "Wei Yang",
        "Deheng Ye"
      ],
      "summary": "The goal of program synthesis, or code generation, is to generate executable code based on given descriptions. Recently, there has been an increasing number of studies employing reinforcement learning (RL) to improve the performance of large language models (LLMs) for code. However, current representative works either rely solely on offline frameworks, limiting the exploration of new sample spaces, or fall short in the utilization of unit test signals, not accounting for specific error locations within the code. To address these issues, we propose RLTF, i.e., Reinforcement Learning from Unit Test Feedback, a novel online RL framework with unit test feedback of multi-granularity for refining code LLMs. Our approach generates data in real-time during training and simultaneously utilizes fine-grained feedback signals to guide the model towards producing higher-quality code. Extensive experiments show that RLTF achieves state-of-the-art performance on the APPS and the MBPP benchmarks. Our code is available at: https://github.com/Zyq-scut/RLTF.",
      "arxiv_id": "2307.04349v2",
      "published": "2023-07-10",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2307.04349v2",
      "pdf_url": "https://arxiv.org/pdf/2307.04349v2.pdf"
    }
  ],
  "llm_alignment": [
    {
      "title": "Multi-Preference Lambda-weighted Listwise DPO for Small-Scale Model Alignment",
      "authors": [
        "Yuhui Sun",
        "Xiyao Wang",
        "Zixi Li",
        "Zhenlong Yuan",
        "Jinman Zhao"
      ],
      "summary": "Large language models (LLMs) demonstrate strong generalization across a wide range of language tasks, but often generate outputs that misalign with human preferences. Reinforcement Learning from Human Feedback (RLHF) addresses this by optimizing models toward human preferences using a learned reward function and reinforcement learning, yielding improved alignment but suffering from high computational cost and instability. Direct Preference Optimization (DPO) simplifies the process by treating alignment as a classification task over binary preference pairs, reducing training overhead while achieving competitive performance. However, it assumes fixed, single-dimensional preferences and only supports pairwise supervision.   To address these limitations, we propose Multi-Preference Lambda-weighted Listwise DPO, which allows the model to learn from more detailed human feedback and flexibly balance multiple goals such as helpfulness, honesty, and fluency. Our method models full-ranked preference distributions rather than binary comparisons, enabling more informative learning signals. The lambda vector controls the relative importance of different alignment goals, allowing the model to generalize across diverse human objectives. During inference, lambda can be adjusted without retraining, providing controllable alignment behavior for downstream use. We also introduce a learned scheduler that dynamically samples performant lambda configurations to improve robustness.   Notably, our method requires only 20GB of GPU memory for training, making it suitable for compute-constrained settings such as academic labs, educational tools, or on-device assistants. Experiments on 1B-2B scale models show that our method consistently outperforms standard DPO on alignment benchmarks while enabling efficient, controllable, and fine-grained adaptation suitable for real-world deployment.",
      "arxiv_id": "2506.19780v5",
      "published": "2025-06-24",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2506.19780v5",
      "pdf_url": "https://arxiv.org/pdf/2506.19780v5.pdf"
    },
    {
      "title": "Data-Centric Human Preference with Rationales for Direct Preference Alignment",
      "authors": [
        "Hoang Anh Just",
        "Ming Jin",
        "Anit Sahu",
        "Huy Phan",
        "Ruoxi Jia"
      ],
      "summary": "Aligning language models with human preferences through reinforcement learning from human feedback is crucial for their safe and effective deployment. The human preference is typically represented through comparison where one response is chosen over another for a given prompt. However, standard preference datasets often lack explicit information on why a particular choice was made, presenting an ambiguity that can hinder efficient learning and robust alignment, especially given the high cost of acquiring extensive human annotations. While many studies focus on algorithmic improvements, this work adopts a data-centric perspective, exploring how to enhance learning from existing preference data. We propose augmenting standard preference pairs with rationales that explain the reasoning behind the human preference. Specifically, we introduce a simple and principled framework that leverages machine-generated rationales to enrich preference data for preference optimization algorithms. Our comprehensive analysis demonstrates that incorporating rationales improves learning efficiency. Extensive experiments reveal some advantages: rationale-augmented learning accelerates convergence and can achieve higher final model performance. Furthermore, this approach is versatile and compatible with various direct preference optimization algorithms. Our findings showcase the potential of thoughtful data design in preference learning, demonstrating that enriching existing datasets with explanatory rationales can help unlock improvements in model alignment and annotation efficiency.",
      "arxiv_id": "2407.14477v4",
      "published": "2024-07-19",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2407.14477v4",
      "pdf_url": "https://arxiv.org/pdf/2407.14477v4.pdf"
    },
    {
      "title": "Active Learning for Direct Preference Optimization",
      "authors": [
        "Branislav Kveton",
        "Xintong Li",
        "Julian McAuley",
        "Ryan Rossi",
        "Jingbo Shang",
        "Junda Wu",
        "Tong Yu"
      ],
      "summary": "Direct preference optimization (DPO) is a form of reinforcement learning from human feedback (RLHF) where the policy is learned directly from preferential feedback. Although many models of human preferences exist, the critical task of selecting the most informative feedback for training them is under-explored. We propose an active learning framework for DPO, which can be applied to collect human feedback online or to choose the most informative subset of already collected feedback offline. We propose efficient algorithms for both settings. The key idea is to linearize the DPO objective at the last layer of the neural network representation of the optimized policy and then compute the D-optimal design to collect preferential feedback. We prove that the errors in our DPO logit estimates diminish with more feedback. We show the effectiveness of our algorithms empirically in the setting that matches our theory and also on large language models.",
      "arxiv_id": "2503.01076v1",
      "published": "2025-03-03",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2503.01076v1",
      "pdf_url": "https://arxiv.org/pdf/2503.01076v1.pdf"
    },
    {
      "title": "Reinforcement Learning from Human Feedback with Active Queries",
      "authors": [
        "Kaixuan Ji",
        "Jiafan He",
        "Quanquan Gu"
      ],
      "summary": "Aligning large language models (LLM) with human preference plays a key role in building modern generative models and can be achieved by reinforcement learning from human feedback (RLHF). Despite their superior performance, current RLHF approaches often require a large amount of human-labelled preference data, which is expensive to collect. In this paper, inspired by the success of active learning, we address this problem by proposing query-efficient RLHF methods. We first formalize the alignment problem as a contextual dueling bandit problem and design an active-query-based proximal policy optimization (APPO) algorithm with an $\\tilde{O}(d^2/\u0394)$ instance-dependent regret bound and an $\\tilde{O}(d^2/\u0394^2)$ query complexity, where $d$ is the dimension of feature space and $\u0394$ is the sub-optimality gap over all the contexts. We then propose ADPO, a practical version of our algorithm based on direct preference optimization (DPO) and apply it to fine-tuning LLMs. Our experiments show that ADPO, while only making about half of queries for human preference, matches the performance of the state-of-the-art DPO method.",
      "arxiv_id": "2402.09401v2",
      "published": "2024-02-14",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "math.OC",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2402.09401v2",
      "pdf_url": "https://arxiv.org/pdf/2402.09401v2.pdf"
    },
    {
      "title": "Difficulty-Based Preference Data Selection by DPO Implicit Reward Gap",
      "authors": [
        "Xuan Qi",
        "Rongwu Xu",
        "Zhijing Jin"
      ],
      "summary": "Aligning large language models (LLMs) with human preferences is a critical challenge in AI research. While methods like Reinforcement Learning from Human Feedback (RLHF) and Direct Preference Optimization (DPO) are widely used, they often rely on large, costly preference datasets. The current work lacks methods for high-quality data selection specifically for preference data. In this work, we introduce a novel difficulty-based data selection strategy for preference datasets, grounded in the DPO implicit reward mechanism. By selecting preference data examples with smaller DPO implicit reward gaps, which are indicative of more challenging cases, we improve data efficiency and model alignment. Our approach consistently outperforms five strong baselines across multiple datasets and alignment tasks, achieving superior performance with only 10\\% of the original data. This principled, efficient selection method offers a promising solution for scaling LLM alignment with limited resources.",
      "arxiv_id": "2508.04149v1",
      "published": "2025-08-06",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2508.04149v1",
      "pdf_url": "https://arxiv.org/pdf/2508.04149v1.pdf"
    },
    {
      "title": "Multi-turn Reinforcement Learning from Preference Human Feedback",
      "authors": [
        "Lior Shani",
        "Aviv Rosenberg",
        "Asaf Cassel",
        "Oran Lang",
        "Daniele Calandriello",
        "Avital Zipori",
        "Hila Noga",
        "Orgad Keller",
        "Bilal Piot",
        "Idan Szpektor",
        "Avinatan Hassidim",
        "Yossi Matias",
        "R\u00e9mi Munos"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has become the standard approach for aligning Large Language Models (LLMs) with human preferences, allowing LLMs to demonstrate remarkable abilities in various tasks. Existing methods work by emulating the preferences at the single decision (turn) level, limiting their capabilities in settings that require planning or multi-turn interactions to achieve a long-term goal. In this paper, we address this issue by developing novel methods for Reinforcement Learning (RL) from preference feedback between two full multi-turn conversations. In the tabular setting, we present a novel mirror-descent-based policy optimization algorithm for the general multi-turn preference-based RL problem, and prove its convergence to Nash equilibrium. To evaluate performance, we create a new environment, Education Dialogue, where a teacher agent guides a student in learning a random topic, and show that a deep RL variant of our algorithm outperforms RLHF baselines. Finally, we show that in an environment with explicit rewards, our algorithm recovers the same performance as a reward-based RL baseline, despite relying solely on a weaker preference signal.",
      "arxiv_id": "2405.14655v2",
      "published": "2024-05-23",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.14655v2",
      "pdf_url": "https://arxiv.org/pdf/2405.14655v2.pdf"
    },
    {
      "title": "TIS-DPO: Token-level Importance Sampling for Direct Preference Optimization With Estimated Weights",
      "authors": [
        "Aiwei Liu",
        "Haoping Bai",
        "Zhiyun Lu",
        "Yanchao Sun",
        "Xiang Kong",
        "Simon Wang",
        "Jiulong Shan",
        "Albin Madappally Jose",
        "Xiaojiang Liu",
        "Lijie Wen",
        "Philip S. Yu",
        "Meng Cao"
      ],
      "summary": "Direct Preference Optimization (DPO) has been widely adopted for preference alignment of Large Language Models (LLMs) due to its simplicity and effectiveness. However, DPO is derived as a bandit problem in which the whole response is treated as a single arm, ignoring the importance differences between tokens, which may affect optimization efficiency and make it difficult to achieve optimal results. In this work, we propose that the optimal data for DPO has equal expected rewards for each token in winning and losing responses, as there is no difference in token importance. However, since the optimal dataset is unavailable in practice, we propose using the original dataset for importance sampling to achieve unbiased optimization. Accordingly, we propose a token-level importance sampling DPO objective named TIS-DPO that assigns importance weights to each token based on its reward. Inspired by previous works, we estimate the token importance weights using the difference in prediction probabilities from a pair of contrastive LLMs. We explore three methods to construct these contrastive LLMs: (1) guiding the original LLM with contrastive prompts, (2) training two separate LLMs using winning and losing responses, and (3) performing forward and reverse DPO training with winning and losing responses. Experiments show that TIS-DPO significantly outperforms various baseline methods on harmlessness and helpfulness alignment and summarization tasks. We also visualize the estimated weights, demonstrating their ability to identify key token positions.",
      "arxiv_id": "2410.04350v3",
      "published": "2024-10-06",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2410.04350v3",
      "pdf_url": "https://arxiv.org/pdf/2410.04350v3.pdf"
    },
    {
      "title": "PILAF: Optimal Human Preference Sampling for Reward Modeling",
      "authors": [
        "Yunzhen Feng",
        "Ariel Kwiatkowski",
        "Kunhao Zheng",
        "Julia Kempe",
        "Yaqi Duan"
      ],
      "summary": "As large language models increasingly drive real-world applications, aligning them with human values becomes paramount. Reinforcement Learning from Human Feedback (RLHF) has emerged as a key technique, translating preference data into reward models when oracle human values remain inaccessible. In practice, RLHF mostly relies on approximate reward models, which may not consistently guide the policy toward maximizing the underlying human values. We propose Policy-Interpolated Learning for Aligned Feedback (PILAF), a novel response sampling strategy for preference labeling that explicitly aligns preference learning with maximizing the underlying oracle reward. PILAF is theoretically grounded, demonstrating optimality from both an optimization and a statistical perspective. The method is straightforward to implement and demonstrates strong performance in iterative and online RLHF settings where feedback curation is critical.",
      "arxiv_id": "2502.04270v1",
      "published": "2025-02-06",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2502.04270v1",
      "pdf_url": "https://arxiv.org/pdf/2502.04270v1.pdf"
    },
    {
      "title": "Sharpe Ratio-Guided Active Learning for Preference Optimization in RLHF",
      "authors": [
        "Syrine Belakaria",
        "Joshua Kazdan",
        "Charles Marx",
        "Chris Cundy",
        "Willie Neiswanger",
        "Sanmi Koyejo",
        "Barbara E. Engelhardt",
        "Stefano Ermon"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has become a cornerstone of the training and alignment pipeline for large language models (LLMs). Recent advances, such as direct preference optimization (DPO), have simplified the preference learning step. However, collecting preference data remains a challenging and costly process, often requiring expert annotation. This cost can be mitigated by carefully selecting the data points presented for annotation. In this work, we propose an active learning approach to efficiently select prompt and preference pairs using a risk assessment strategy based on the Sharpe Ratio. To address the challenge of unknown preferences prior to annotation, our method evaluates the gradients of all potential preference annotations to assess their impact on model updates. These gradient-based evaluations enable risk assessment of data points regardless of the annotation outcome. By leveraging the DPO loss derivations, we derive a closed-form expression for computing these Sharpe ratios on a per-tuple basis, ensuring our approach remains both tractable and computationally efficient. We also introduce two variants of our method, each making different assumptions about prior information. Experimental results demonstrate that our method outperforms the baseline by up to 5% in win rates against the chosen completion with limited human preference data across several language models and real-world datasets.",
      "arxiv_id": "2503.22137v1",
      "published": "2025-03-28",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.22137v1",
      "pdf_url": "https://arxiv.org/pdf/2503.22137v1.pdf"
    },
    {
      "title": "Step-DPO: Step-wise Preference Optimization for Long-chain Reasoning of LLMs",
      "authors": [
        "Xin Lai",
        "Zhuotao Tian",
        "Yukang Chen",
        "Senqiao Yang",
        "Xiangru Peng",
        "Jiaya Jia"
      ],
      "summary": "Mathematical reasoning presents a significant challenge for Large Language Models (LLMs) due to the extensive and precise chain of reasoning required for accuracy. Ensuring the correctness of each reasoning step is critical. To address this, we aim to enhance the robustness and factuality of LLMs by learning from human feedback. However, Direct Preference Optimization (DPO) has shown limited benefits for long-chain mathematical reasoning, as models employing DPO struggle to identify detailed errors in incorrect answers. This limitation stems from a lack of fine-grained process supervision. We propose a simple, effective, and data-efficient method called Step-DPO, which treats individual reasoning steps as units for preference optimization rather than evaluating answers holistically. Additionally, we have developed a data construction pipeline for Step-DPO, enabling the creation of a high-quality dataset containing 10K step-wise preference pairs. We also observe that in DPO, self-generated data is more effective than data generated by humans or GPT-4, due to the latter's out-of-distribution nature. Our findings demonstrate that as few as 10K preference data pairs and fewer than 500 Step-DPO training steps can yield a nearly 3% gain in accuracy on MATH for models with over 70B parameters. Notably, Step-DPO, when applied to Qwen2-72B-Instruct, achieves scores of 70.8% and 94.0% on the test sets of MATH and GSM8K, respectively, surpassing a series of closed-source models, including GPT-4-1106, Claude-3-Opus, and Gemini-1.5-Pro. Our code, data, and models are available at https://github.com/dvlab-research/Step-DPO.",
      "arxiv_id": "2406.18629v1",
      "published": "2024-06-26",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2406.18629v1",
      "pdf_url": "https://arxiv.org/pdf/2406.18629v1.pdf"
    },
    {
      "title": "Contrastive Preference Learning: Learning from Human Feedback without RL",
      "authors": [
        "Joey Hejna",
        "Rafael Rafailov",
        "Harshit Sikchi",
        "Chelsea Finn",
        "Scott Niekum",
        "W. Bradley Knox",
        "Dorsa Sadigh"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a popular paradigm for aligning models with human intent. Typically RLHF algorithms operate in two phases: first, use human preferences to learn a reward function and second, align the model by optimizing the learned reward via reinforcement learning (RL). This paradigm assumes that human preferences are distributed according to reward, but recent work suggests that they instead follow the regret under the user's optimal policy. Thus, learning a reward function from feedback is not only based on a flawed assumption of human preference, but also leads to unwieldy optimization challenges that stem from policy gradients or bootstrapping in the RL phase. Because of these optimization challenges, contemporary RLHF methods restrict themselves to contextual bandit settings (e.g., as in large language models) or limit observation dimensionality (e.g., state-based robotics). We overcome these limitations by introducing a new family of algorithms for optimizing behavior from human feedback using the regret-based model of human preferences. Using the principle of maximum entropy, we derive Contrastive Preference Learning (CPL), an algorithm for learning optimal policies from preferences without learning reward functions, circumventing the need for RL. CPL is fully off-policy, uses only a simple contrastive objective, and can be applied to arbitrary MDPs. This enables CPL to elegantly scale to high-dimensional and sequential RLHF problems while being simpler than prior methods.",
      "arxiv_id": "2310.13639v3",
      "published": "2023-10-20",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2310.13639v3",
      "pdf_url": "https://arxiv.org/pdf/2310.13639v3.pdf"
    },
    {
      "title": "Zeroth-Order Policy Gradient for Reinforcement Learning from Human Feedback without Reward Inference",
      "authors": [
        "Qining Zhang",
        "Lei Ying"
      ],
      "summary": "Reward inference (learning a reward model from human preferences) is a critical intermediate step in the Reinforcement Learning from Human Feedback (RLHF) pipeline for fine-tuning Large Language Models (LLMs). In practice, RLHF faces fundamental challenges such as distribution shift, reward model overfitting, and problem misspecification. An alternative approach is direct policy optimization without reward inference, such as Direct Preference Optimization (DPO), which provides a much simpler pipeline and has shown empirical success in LLM applications. However, DPO utilizes the closed-form expression between the optimal policy and the reward function, which is only suitable under the bandit setting or deterministic MDPs. This paper develops two RLHF algorithms without reward inference for general RL problems beyond bandits and deterministic MDPs, and general preference models beyond the Bradley-Terry model. The key idea is to estimate the local value function difference from human preferences and then approximate the policy gradient with a zeroth-order gradient approximator. For both algorithms, we establish polynomial convergence rates in terms of the number of policy gradient iterations, the number of trajectory samples, and human preference queries per iteration. Numerical experiments in stochastic environments validate the performance of our proposed algorithms, outperforming popular RLHF baselines such as DPO and PPO. Our paper shows there exist provably efficient methods to solve general RLHF problems without reward inference.",
      "arxiv_id": "2409.17401v2",
      "published": "2024-09-25",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2409.17401v2",
      "pdf_url": "https://arxiv.org/pdf/2409.17401v2.pdf"
    },
    {
      "title": "A Unified Theoretical Analysis of Private and Robust Offline Alignment: from RLHF to DPO",
      "authors": [
        "Xingyu Zhou",
        "Yulian Wu",
        "Francesco Orabona"
      ],
      "summary": "In this paper, we theoretically investigate the effects of noisy labels in offline alignment, with a focus on the interplay between privacy and robustness against adversarial corruption. Specifically, under linear modeling assumptions, we present a unified analysis covering both reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO) under different privacy-corruption scenarios, such as Local differential privacy-then-Corruption (LTC), where human preference labels are privatized before being corrupted by an adversary, and Corruption-then-Local differential privacy (CTL), where labels are corrupted before privacy protection. Our analysis leverages a reduction framework that reduces the offline alignment problem under linear modeling assumptions to parameter estimation in logistic regression. This framework allows us to establish an interesting separation result between LTC and CTL, demonstrating that LTC presents a greater challenge than CTL in offline alignment, even under linear models. As important by-products, our findings also advance the state-of-the-art theoretical results in offline alignment under privacy-only or corruption-only scenarios.",
      "arxiv_id": "2505.15694v1",
      "published": "2025-05-21",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.15694v1",
      "pdf_url": "https://arxiv.org/pdf/2505.15694v1.pdf"
    },
    {
      "title": "Dual Active Learning for Reinforcement Learning from Human Feedback",
      "authors": [
        "Pangpang Liu",
        "Chengchun Shi",
        "Will Wei Sun"
      ],
      "summary": "Aligning large language models (LLMs) with human preferences is critical to recent advances in generative artificial intelligence. Reinforcement learning from human feedback (RLHF) is widely applied to achieve this objective. A key step in RLHF is to learn the reward function from human feedback. However, human feedback is costly and time-consuming, making it essential to collect high-quality conversation data for human teachers to label. Additionally, different human teachers have different levels of expertise. It is thus critical to query the most appropriate teacher for their opinions. In this paper, we use offline reinforcement learning (RL) to formulate the alignment problem. Motivated by the idea of $D$-optimal design, we first propose a dual active reward learning algorithm for the simultaneous selection of conversations and teachers. Next, we apply pessimistic RL to solve the alignment problem, based on the learned reward estimator. Theoretically, we show that the reward estimator obtained through our proposed adaptive selection strategy achieves minimal generalized variance asymptotically, and prove that the sub-optimality of our pessimistic policy scales as $O(1/\\sqrt{T})$ with a given sample budget $T$. Through simulations and experiments on LLMs, we demonstrate the effectiveness of our algorithm and its superiority over state-of-the-arts.",
      "arxiv_id": "2410.02504v2",
      "published": "2024-10-03",
      "categories": [
        "stat.ML",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.02504v2",
      "pdf_url": "https://arxiv.org/pdf/2410.02504v2.pdf"
    },
    {
      "title": "MaxMin-RLHF: Alignment with Diverse Human Preferences",
      "authors": [
        "Souradip Chakraborty",
        "Jiahao Qiu",
        "Hui Yuan",
        "Alec Koppel",
        "Furong Huang",
        "Dinesh Manocha",
        "Amrit Singh Bedi",
        "Mengdi Wang"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) aligns language models to human preferences by employing a singular reward model derived from preference data. However, such an approach overlooks the rich diversity of human preferences inherent in data collected from multiple users. In this work, we first derive an impossibility result of alignment with single reward RLHF, thereby highlighting its insufficiency in representing diverse human preferences. To provide an equitable solution to the problem, we learn a mixture of preference distributions via an expectation-maximization algorithm and propose a MaxMin alignment objective for policy learning inspired by the Egalitarian principle in social choice theory to better represent diverse human preferences. We elucidate the connection of our proposed approach to distributionally robust optimization and general utility RL, thereby highlighting the generality and robustness of our proposed solution. We present comprehensive experimental results on small-scale (GPT-2) and large-scale language models (with Tulu2-7B) and show the efficacy of the proposed approach in the presence of diversity among human preferences. Our algorithm achieves an average improvement of more than 16% in win-rates over conventional RLHF algorithms and improves the win-rate (accuracy) for minority groups by over 33% without compromising the performance of majority groups, showcasing the robustness and fairness of our approach. We remark that our findings in this work are not only limited to language models but also extend to reinforcement learning in general.",
      "arxiv_id": "2402.08925v2",
      "published": "2024-02-14",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2402.08925v2",
      "pdf_url": "https://arxiv.org/pdf/2402.08925v2.pdf"
    },
    {
      "title": "Reward Model Learning vs. Direct Policy Optimization: A Comparative Analysis of Learning from Human Preferences",
      "authors": [
        "Andi Nika",
        "Debmalya Mandal",
        "Parameswaran Kamalaruban",
        "Georgios Tzannetos",
        "Goran Radanovi\u0107",
        "Adish Singla"
      ],
      "summary": "In this paper, we take a step towards a deeper understanding of learning from human preferences by systematically comparing the paradigm of reinforcement learning from human feedback (RLHF) with the recently proposed paradigm of direct preference optimization (DPO). We focus our attention on the class of loglinear policy parametrization and linear reward functions. In order to compare the two paradigms, we first derive minimax statistical bounds on the suboptimality gap induced by both RLHF and DPO, assuming access to an oracle that exactly solves the optimization problems. We provide a detailed discussion on the relative comparison between the two paradigms, simultaneously taking into account the sample size, policy and reward class dimensions, and the regularization temperature. Moreover, we extend our analysis to the approximate optimization setting and derive exponentially decaying convergence rates for both RLHF and DPO. Next, we analyze the setting where the ground-truth reward is not realizable and find that, while RLHF incurs a constant additional error, DPO retains its asymptotically decaying gap by just tuning the temperature accordingly. Finally, we extend our comparison to the Markov decision process setting, where we generalize our results with exact optimization. To the best of our knowledge, we are the first to provide such a comparative analysis for RLHF and DPO.",
      "arxiv_id": "2403.01857v2",
      "published": "2024-03-04",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2403.01857v2",
      "pdf_url": "https://arxiv.org/pdf/2403.01857v2.pdf"
    },
    {
      "title": "Robust Reinforcement Learning from Corrupted Human Feedback",
      "authors": [
        "Alexander Bukharin",
        "Ilgee Hong",
        "Haoming Jiang",
        "Zichong Li",
        "Qingru Zhang",
        "Zixuan Zhang",
        "Tuo Zhao"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) provides a principled framework for aligning AI systems with human preference data. For various reasons, e.g., personal bias, context ambiguity, lack of training, etc, human annotators may give incorrect or inconsistent preference labels. To tackle this challenge, we propose a robust RLHF approach -- $R^3M$, which models the potentially corrupted preference label as sparse outliers. Accordingly, we formulate the robust reward learning as an $\\ell_1$-regularized maximum likelihood estimation problem. Computationally, we develop an efficient alternating optimization algorithm, which only incurs negligible computational overhead compared with the standard RLHF approach. Theoretically, we prove that under proper regularity conditions, $R^3M$ can consistently learn the underlying reward and identify outliers, provided that the number of outlier labels scales sublinearly with the preference sample size. Furthermore, we remark that $R^3M$ is versatile and can be extended to various preference optimization methods, including direct preference optimization (DPO). Our experiments on robotic control and natural language generation with large language models (LLMs) show that $R^3M$ improves robustness of the reward against several types of perturbations to the preference data.",
      "arxiv_id": "2406.15568v2",
      "published": "2024-06-21",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2406.15568v2",
      "pdf_url": "https://arxiv.org/pdf/2406.15568v2.pdf"
    },
    {
      "title": "Multi-Task Reward Learning from Human Ratings",
      "authors": [
        "Mingkang Wu",
        "Devin White",
        "Evelyn Rose",
        "Vernon Lawhern",
        "Nicholas R Waytowich",
        "Yongcan Cao"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has become a key factor in aligning model behavior with users' goals. However, while humans integrate multiple strategies when making decisions, current RLHF approaches often simplify this process by modeling human reasoning through isolated tasks such as classification or regression. In this paper, we propose a novel reinforcement learning (RL) method that mimics human decision-making by jointly considering multiple tasks. Specifically, we leverage human ratings in reward-free environments to infer a reward function, introducing learnable weights that balance the contributions of both classification and regression models. This design captures the inherent uncertainty in human decision-making and allows the model to adaptively emphasize different strategies. We conduct several experiments using synthetic human ratings to validate the effectiveness of the proposed approach. Results show that our method consistently outperforms existing rating-based RL methods, and in some cases, even surpasses traditional RL approaches.",
      "arxiv_id": "2506.09183v2",
      "published": "2025-06-10",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.09183v2",
      "pdf_url": "https://arxiv.org/pdf/2506.09183v2.pdf"
    },
    {
      "title": "Humans are not Boltzmann Distributions: Challenges and Opportunities for Modelling Human Feedback and Interaction in Reinforcement Learning",
      "authors": [
        "David Lindner",
        "Mennatallah El-Assady"
      ],
      "summary": "Reinforcement learning (RL) commonly assumes access to well-specified reward functions, which many practical applications do not provide. Instead, recently, more work has explored learning what to do from interacting with humans. So far, most of these approaches model humans as being (nosily) rational and, in particular, giving unbiased feedback. We argue that these models are too simplistic and that RL researchers need to develop more realistic human models to design and evaluate their algorithms. In particular, we argue that human models have to be personal, contextual, and dynamic. This paper calls for research from different disciplines to address key questions about how humans provide feedback to AIs and how we can build more robust human-in-the-loop RL systems.",
      "arxiv_id": "2206.13316v1",
      "published": "2022-06-27",
      "categories": [
        "cs.LG",
        "cs.HC",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2206.13316v1",
      "pdf_url": "https://arxiv.org/pdf/2206.13316v1.pdf"
    },
    {
      "title": "Personalizing Reinforcement Learning from Human Feedback with Variational Preference Learning",
      "authors": [
        "Sriyash Poddar",
        "Yanming Wan",
        "Hamish Ivison",
        "Abhishek Gupta",
        "Natasha Jaques"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is a powerful paradigm for aligning foundation models to human values and preferences. However, current RLHF techniques cannot account for the naturally occurring differences in individual human preferences across a diverse population. When these differences arise, traditional RLHF frameworks simply average over them, leading to inaccurate rewards and poor performance for individual subgroups. To address the need for pluralistic alignment, we develop a class of multimodal RLHF methods. Our proposed techniques are based on a latent variable formulation - inferring a novel user-specific latent and learning reward models and policies conditioned on this latent without additional user-specific data. While conceptually simple, we show that in practice, this reward modeling requires careful algorithmic considerations around model architecture and reward scaling. To empirically validate our proposed technique, we first show that it can provide a way to combat underspecification in simulated control problems, inferring and optimizing user-specific reward functions. Next, we conduct experiments on pluralistic language datasets representing diverse user preferences and demonstrate improved reward function accuracy. We additionally show the benefits of this probabilistic framework in terms of measuring uncertainty, and actively learning user preferences. This work enables learning from diverse populations of users with divergent preferences, an important challenge that naturally occurs in problems from robot learning to foundation model alignment.",
      "arxiv_id": "2408.10075v1",
      "published": "2024-08-19",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2408.10075v1",
      "pdf_url": "https://arxiv.org/pdf/2408.10075v1.pdf"
    },
    {
      "title": "MA-RLHF: Reinforcement Learning from Human Feedback with Macro Actions",
      "authors": [
        "Yekun Chai",
        "Haoran Sun",
        "Huang Fang",
        "Shuohuan Wang",
        "Yu Sun",
        "Hua Wu"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has demonstrated effectiveness in aligning large language models (LLMs) with human preferences. However, token-level RLHF suffers from the credit assignment problem over long sequences, where delayed rewards make it challenging for the model to discern which actions contributed to preferred outcomes. This hinders learning efficiency and slows convergence.In this paper, we propose MA-RLHF, a simple yet effective RLHF framework that incorporates macro actions -- sequences of tokens or higher-level language constructs -- into the learning process. By operating at higher level of abstraction, our approach reduces the temporal distance between actions and rewards, facilitating faster and more accurate credit assignment. This results in more stable policy gradient estimates and enhances learning efficiency within each episode, all without increasing computational complexity during training or inference. We validate our approach through extensive experiments across various model sizes and tasks, including text summarization, dialogue generation, question answering, and program synthesis. Our method achieves substantial performance improvements over standard RLHF, with performance gains of up to 30% in text summarization and code generation, 18% in dialogue, and 8% in question answering tasks. Notably, our approach reaches parity with vanilla RLHF 1.7 ~ 2 times faster in terms of training time and continues to outperform it with further training. We make our code and data publicly available at https://github.com/ernie-research/MA-RLHF.",
      "arxiv_id": "2410.02743v2",
      "published": "2024-10-03",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2410.02743v2",
      "pdf_url": "https://arxiv.org/pdf/2410.02743v2.pdf"
    },
    {
      "title": "2D-Curri-DPO: Two-Dimensional Curriculum Learning for Direct Preference Optimization",
      "authors": [
        "Mengyang Li",
        "Zhong Zhang"
      ],
      "summary": "Aligning large language models with human preferences is crucial for their safe deployment. While Direct Preference Optimization (DPO) offers an efficient alternative to reinforcement learning from human feedback, traditional DPO methods are limited by their reliance on single preference pairs. Recent work like Curriculum-DPO integrates multiple pairs using a one-dimensional difficulty curriculum based on pairwise distinguishability (PD), but overlooks the complexity of the input prompt itself. To address this, we propose 2D-Curri-DPO, a novel framework employing a two-dimensional curriculum that jointly models Prompt Complexity (PC) and Pairwise Distinguishability. This framework introduces dual difficulty metrics to quantify prompt semantic complexity and response preference clarity, defines a curriculum strategy space encompassing multiple selectable strategies for task adaptation, and incorporates a KL-divergence-based adaptive mechanism for dynamic reference model updates to enhance training stability. Comprehensive experiments demonstrate that 2D-Curri-DPO significantly outperforms standard DPO and prior curriculum methods across multiple benchmarks, including MT-Bench, Vicuna Bench, and WizardLM. Our approach achieves state-of-the-art performance on challenging test sets like UltraFeedback. Ablation studies confirm the benefits of the 2D structure and adaptive mechanisms, while analysis provides guidance for strategy selection. These findings demonstrate that effective alignment requires modeling both prompt complexity and pairwise distinguishability, establishing adaptive, multi-dimensional curriculum learning as a powerful and interpretable new paradigm for preference-based language model optimization.",
      "arxiv_id": "2504.07856v3",
      "published": "2025-04-10",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2504.07856v3",
      "pdf_url": "https://arxiv.org/pdf/2504.07856v3.pdf"
    },
    {
      "title": "Provably Mitigating Corruption, Overoptimization, and Verbosity Simultaneously in Offline and Online RLHF/DPO Alignment",
      "authors": [
        "Ziyi Chen",
        "Junyi Li",
        "Peiran Yu",
        "Heng Huang"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) and direct preference optimization (DPO) are important techniques to align large language models (LLM) with human preference. However, the quality of RLHF and DPO training is seriously compromised by \\textit{\\textbf{C}orrupted} preference, reward \\textit{\\textbf{O}veroptimization}, and bias towards \\textit{\\textbf{V}erbosity}. To our knowledge, most existing works tackle only one of these important issues, and the few other works require much computation to estimate multiple reward models and lack theoretical guarantee of generalization ability. In this work, we propose RLHF-\\textbf{COV} and DPO-\\textbf{COV} algorithms that can simultaneously mitigate these three issues, in both offline and online settings. This ability is theoretically demonstrated by obtaining length-regularized generalization error rates for our DPO-COV algorithms trained on corrupted data, which match the best-known rates for simpler cases with clean data and without length regularization. Moreover, our DPO-COV algorithm is simple to implement without reward estimation, and is proved to be equivalent to our RLHF-COV algorithm, which directly implies the equivalence between the vanilla RLHF and DPO algorithms. Experiments demonstrate the effectiveness of our DPO-COV algorithms under both offline and online settings.",
      "arxiv_id": "2510.05526v1",
      "published": "2025-10-07",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.05526v1",
      "pdf_url": "https://arxiv.org/pdf/2510.05526v1.pdf"
    },
    {
      "title": "Back to Basics: Revisiting REINFORCE Style Optimization for Learning from Human Feedback in LLMs",
      "authors": [
        "Arash Ahmadian",
        "Chris Cremer",
        "Matthias Gall\u00e9",
        "Marzieh Fadaee",
        "Julia Kreutzer",
        "Olivier Pietquin",
        "Ahmet \u00dcst\u00fcn",
        "Sara Hooker"
      ],
      "summary": "AI alignment in the shape of Reinforcement Learning from Human Feedback (RLHF) is increasingly treated as a crucial ingredient for high performance large language models. Proximal Policy Optimization (PPO) has been positioned by recent literature as the canonical method for the RL part of RLHF. However, it involves both high computational cost and sensitive hyperparameter tuning. We posit that most of the motivational principles that led to the development of PPO are less of a practical concern in RLHF and advocate for a less computationally expensive method that preserves and even increases performance. We revisit the formulation of alignment from human preferences in the context of RL. Keeping simplicity as a guiding principle, we show that many components of PPO are unnecessary in an RLHF context and that far simpler REINFORCE-style optimization variants outperform both PPO and newly proposed \"RL-free\" methods such as DPO and RAFT. Our work suggests that careful adaptation to LLMs alignment characteristics enables benefiting from online RL optimization at low cost.",
      "arxiv_id": "2402.14740v2",
      "published": "2024-02-22",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2402.14740v2",
      "pdf_url": "https://arxiv.org/pdf/2402.14740v2.pdf"
    },
    {
      "title": "Disentangling Length from Quality in Direct Preference Optimization",
      "authors": [
        "Ryan Park",
        "Rafael Rafailov",
        "Stefano Ermon",
        "Chelsea Finn"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has been a crucial component in the recent success of Large Language Models. However, RLHF is know to exploit biases in human preferences, such as verbosity. A well-formatted and eloquent answer is often more highly rated by users, even when it is less helpful and objective. A number of approaches have been developed to control those biases in the classical RLHF literature, but the problem remains relatively under-explored for Direct Alignment Algorithms such as Direct Preference Optimization (DPO). Unlike classical RLHF, DPO does not train a separate reward model or use reinforcement learning directly, so previous approaches developed to control verbosity cannot be directly applied to this setting. Our work makes several contributions. For the first time, we study the length problem in the DPO setting, showing significant exploitation in DPO and linking it to out-of-distribution bootstrapping. We then develop a principled but simple regularization strategy that prevents length exploitation, while still maintaining improvements in model quality. We demonstrate these effects across datasets on summarization and dialogue, where we achieve up to 20\\% improvement in win rates when controlling for length, despite the GPT4 judge's well-known verbosity bias.",
      "arxiv_id": "2403.19159v2",
      "published": "2024-03-28",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2403.19159v2",
      "pdf_url": "https://arxiv.org/pdf/2403.19159v2.pdf"
    },
    {
      "title": "MIA-DPO: Multi-Image Augmented Direct Preference Optimization For Large Vision-Language Models",
      "authors": [
        "Ziyu Liu",
        "Yuhang Zang",
        "Xiaoyi Dong",
        "Pan Zhang",
        "Yuhang Cao",
        "Haodong Duan",
        "Conghui He",
        "Yuanjun Xiong",
        "Dahua Lin",
        "Jiaqi Wang"
      ],
      "summary": "Visual preference alignment involves training Large Vision-Language Models (LVLMs) to predict human preferences between visual inputs. This is typically achieved by using labeled datasets of chosen/rejected pairs and employing optimization algorithms like direct preference optimization (DPO). Existing visual alignment methods, primarily designed for single-image scenarios, struggle to effectively handle the complexity of multi-image tasks due to the scarcity of diverse training data and the high cost of annotating chosen/rejected pairs. We present Multi-Image Augmented Direct Preference Optimization (MIA-DPO), a visual preference alignment approach that effectively handles multi-image inputs. MIA-DPO mitigates the scarcity of diverse multi-image training data by extending single-image data with unrelated images arranged in grid collages or pic-in-pic formats, significantly reducing the costs associated with multi-image data annotations. Our observation reveals that attention values of LVLMs vary considerably across different images. We use attention values to identify and filter out rejected responses the model may have mistakenly focused on. Our attention-aware selection for constructing the chosen/rejected pairs without relying on (i) human annotation, (ii) extra data, and (iii) external models or APIs. MIA-DPO is compatible with various architectures and outperforms existing methods on five multi-image benchmarks, achieving an average performance boost of 3.0% on LLaVA-v1.5 and 4.3% on the recent InternLM-XC2.5. Moreover, MIA-DPO has a minimal effect on the model's ability to understand single images.",
      "arxiv_id": "2410.17637v1",
      "published": "2024-10-23",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2410.17637v1",
      "pdf_url": "https://arxiv.org/pdf/2410.17637v1.pdf"
    },
    {
      "title": "Extragradient Preference Optimization (EGPO): Beyond Last-Iterate Convergence for Nash Learning from Human Feedback",
      "authors": [
        "Runlong Zhou",
        "Maryam Fazel",
        "Simon S. Du"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has become essential for improving language model capabilities, but traditional approaches rely on the assumption that human preferences follow a transitive Bradley-Terry model. This assumption fails to capture the non-transitive nature of populational human preferences. Nash learning from human feedback (NLHF), targeting non-transitive preferences, is a problem of computing the Nash equilibrium (NE) of the two-player constant-sum game defined by the human preference. We introduce Extragradient preference optimization (EGPO), a novel algorithm for NLHF achieving last-iterate linear convergence to the NE of KL-regularized games and polynomial convergence to the NE of original games, while being robust to noise. Unlike previous approaches that rely on nested optimization, we derive an equivalent implementation using gradients of an online variant of the identity preference optimization (IPO) loss, enabling more faithful implementation for neural networks. Our empirical evaluations demonstrate EGPO's superior performance over baseline methods when training for the same number of epochs, as measured by pairwise win-rates using the ground truth preference. These results validate both the theoretical strengths and practical advantages of EGPO for language model alignment with non-transitive human preferences.",
      "arxiv_id": "2503.08942v3",
      "published": "2025-03-11",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.08942v3",
      "pdf_url": "https://arxiv.org/pdf/2503.08942v3.pdf"
    },
    {
      "title": "Strategyproof Reinforcement Learning from Human Feedback",
      "authors": [
        "Thomas Kleine Buening",
        "Jiarui Gan",
        "Debmalya Mandal",
        "Marta Kwiatkowska"
      ],
      "summary": "We study Reinforcement Learning from Human Feedback (RLHF) in settings where multiple labelers may strategically misreport feedback to steer the learned policy toward their own preferences. We show that existing RLHF algorithms, including recent pluralistic methods, are not strategyproof, and that even a single strategic labeler can cause arbitrarily large misalignment with social welfare. Moreover, we prove that, in the worst case, any strategyproof RLHF algorithm must perform $k$-times worse than the optimal policy, where $k$ is the number of labelers. This suggests a fundamental trade-off between incentive alignment (ensuring labelers report truthfully) and policy alignment (maximizing social welfare). To address this, we propose the Pessimistic Median of MLEs algorithm, which, under appropriate policy coverage assumptions, is approximately strategyproof and converges to the optimal policy as the number of labelers and samples increases. Our results apply to both contextual bandits and Markov decision processes.",
      "arxiv_id": "2503.09561v2",
      "published": "2025-03-12",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.09561v2",
      "pdf_url": "https://arxiv.org/pdf/2503.09561v2.pdf"
    },
    {
      "title": "Optimizing Autonomous Driving for Safety: A Human-Centric Approach with LLM-Enhanced RLHF",
      "authors": [
        "Yuan Sun",
        "Navid Salami Pargoo",
        "Peter J. Jin",
        "Jorge Ortiz"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is popular in large language models (LLMs), whereas traditional Reinforcement Learning (RL) often falls short. Current autonomous driving methods typically utilize either human feedback in machine learning, including RL, or LLMs. Most feedback guides the car agent's learning process (e.g., controlling the car). RLHF is usually applied in the fine-tuning step, requiring direct human \"preferences,\" which are not commonly used in optimizing autonomous driving models. In this research, we innovatively combine RLHF and LLMs to enhance autonomous driving safety. Training a model with human guidance from scratch is inefficient. Our framework starts with a pre-trained autonomous car agent model and implements multiple human-controlled agents, such as cars and pedestrians, to simulate real-life road environments. The autonomous car model is not directly controlled by humans. We integrate both physical and physiological feedback to fine-tune the model, optimizing this process using LLMs. This multi-agent interactive environment ensures safe, realistic interactions before real-world application. Finally, we will validate our model using data gathered from real-life testbeds located in New Jersey and New York City.",
      "arxiv_id": "2406.04481v1",
      "published": "2024-06-06",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2406.04481v1",
      "pdf_url": "https://arxiv.org/pdf/2406.04481v1.pdf"
    },
    {
      "title": "RS-DPO: A Hybrid Rejection Sampling and Direct Preference Optimization Method for Alignment of Large Language Models",
      "authors": [
        "Saeed Khaki",
        "JinJin Li",
        "Lan Ma",
        "Liu Yang",
        "Prathap Ramachandra"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has been extensively employed to align large language models with user intent. However, proximal policy optimization (PPO) based RLHF is occasionally unstable requiring significant hyperparameter finetuning, and computationally expensive to maximize the estimated reward during alignment. Recently, direct preference optimization (DPO) is proposed to address those challenges. However, DPO relies on contrastive responses generated from human annotator and alternative LLM, instead of the policy model, limiting the effectiveness of the RLHF. In this paper, we addresses both challenges by systematically combining rejection sampling (RS) and DPO. Our proposed method, RS-DPO, initiates with the development of a supervised fine-tuned policy model (SFT). A varied set of k responses per prompt are sampled directly from the SFT model. RS-DPO identifies pairs of contrastive samples based on their reward distribution. Finally, we apply DPO with the contrastive samples to align the model to human preference. Our experiments indicate that our proposed method effectively fine-tunes LLMs with limited resource environments, leading to improved alignment with user intent. Furthermore, it outperforms existing methods, including RS, PPO, and DPO.",
      "arxiv_id": "2402.10038v2",
      "published": "2024-02-15",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.CV",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2402.10038v2",
      "pdf_url": "https://arxiv.org/pdf/2402.10038v2.pdf"
    },
    {
      "title": "Improving Reinforcement Learning from Human Feedback with Efficient Reward Model Ensemble",
      "authors": [
        "Shun Zhang",
        "Zhenfang Chen",
        "Sunli Chen",
        "Yikang Shen",
        "Zhiqing Sun",
        "Chuang Gan"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is a widely adopted approach for aligning large language models with human values. However, RLHF relies on a reward model that is trained with a limited amount of human preference data, which could lead to inaccurate predictions. As a result, RLHF may produce outputs that are misaligned with human values. To mitigate this issue, we contribute a reward ensemble method that allows the reward model to make more accurate predictions. As using an ensemble of large language model-based reward models can be computationally and resource-expensive, we explore efficient ensemble methods including linear-layer ensemble and LoRA-based ensemble. Empirically, we run Best-of-$n$ and Proximal Policy Optimization with our ensembled reward models, and verify that our ensemble methods help improve the alignment performance of RLHF outputs.",
      "arxiv_id": "2401.16635v3",
      "published": "2024-01-30",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2401.16635v3",
      "pdf_url": "https://arxiv.org/pdf/2401.16635v3.pdf"
    },
    {
      "title": "Entropy Controllable Direct Preference Optimization",
      "authors": [
        "Motoki Omura",
        "Yasuhiro Fujita",
        "Toshiki Kataoka"
      ],
      "summary": "In the post-training of large language models (LLMs), Reinforcement Learning from Human Feedback (RLHF) is an effective approach to achieve generation aligned with human preferences. Direct Preference Optimization (DPO) allows for policy training with a simple binary cross-entropy loss without a reward model. The objective of DPO is regularized by reverse KL divergence that encourages mode-seeking fitting to the reference policy. Nonetheless, we indicate that minimizing reverse KL divergence could fail to capture a mode of the reference distribution, which may hurt the policy's performance. Based on this observation, we propose a simple modification to DPO, H-DPO, which allows for control over the entropy of the resulting policy, enhancing the distribution's sharpness and thereby enabling mode-seeking fitting more effectively. In our experiments, we show that H-DPO outperformed DPO across various tasks, demonstrating superior results in pass@$k$ evaluations for mathematical tasks. Moreover, H-DPO is simple to implement, requiring only minor modifications to the loss calculation of DPO, which makes it highly practical and promising for wide-ranging applications in the training of LLMs.",
      "arxiv_id": "2411.07595v2",
      "published": "2024-11-12",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2411.07595v2",
      "pdf_url": "https://arxiv.org/pdf/2411.07595v2.pdf"
    },
    {
      "title": "Omni-DPO: A Dual-Perspective Paradigm for Dynamic Preference Learning of LLMs",
      "authors": [
        "Shangpin Peng",
        "Weinong Wang",
        "Zhuotao Tian",
        "Senqiao Yang",
        "Xing Wu",
        "Haotian Xu",
        "Chengquan Zhang",
        "Takashi Isobe",
        "Baotian Hu",
        "Min Zhang"
      ],
      "summary": "Direct Preference Optimization (DPO) has become a cornerstone of reinforcement learning from human feedback (RLHF) due to its simplicity and efficiency. However, existing DPO-based approaches typically treat all preference pairs uniformly, ignoring critical variations in their inherent quality and learning utility, leading to suboptimal data utilization and performance. To address this challenge, we propose Omni-DPO, a dual-perspective optimization framework that jointly accounts for (1) the inherent quality of each preference pair and (2) the model's evolving performance on those pairs. By adaptively weighting samples according to both data quality and the model's learning dynamics during training, Omni-DPO enables more effective training data utilization and achieves better performance. Experimental results on various models and benchmarks demonstrate the superiority and generalization capabilities of Omni-DPO. On textual understanding tasks, Gemma-2-9b-it finetuned with Omni-DPO beats the leading LLM, Claude 3 Opus, by a significant margin of 6.7 points on the Arena-Hard benchmark. On mathematical reasoning tasks, Omni-DPO consistently outperforms the baseline methods across all benchmarks, providing strong empirical evidence for the effectiveness and robustness of our approach. Code and models will be available at https://github.com/pspdada/Omni-DPO.",
      "arxiv_id": "2506.10054v2",
      "published": "2025-06-11",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2506.10054v2",
      "pdf_url": "https://arxiv.org/pdf/2506.10054v2.pdf"
    },
    {
      "title": "COPR: Continual Learning Human Preference through Optimal Policy Regularization",
      "authors": [
        "Han Zhang",
        "Lin Gui",
        "Yuanzhao Zhai",
        "Hui Wang",
        "Yu Lei",
        "Ruifeng Xu"
      ],
      "summary": "The technique of Reinforcement Learning from Human Feedback (RLHF) is a commonly employed method to improve pre-trained Language Models (LM), enhancing their ability to conform to human preferences. Nevertheless, the current RLHF-based LMs necessitate full retraining each time novel queries or feedback are introduced, which becomes a challenging task because human preferences can vary between different domains or tasks. Retraining LMs poses practical difficulties in many real-world situations due to the significant time and computational resources required, along with concerns related to data privacy. To address this limitation, we propose a new method called Continual Optimal Policy Regularization (COPR), in which we compute the distribution of optimal policy bypassing the partition function and then regularize the current policy based on the historically optimal distribution to mitigate Catastrophic Forgetting (CF). COPR involves a single learning phase and doesn't necessitate complex reinforcement learning. Importantly, it shares the capability with RLHF to learn from unlabeled data by maintaining a scoring module, similar to reward model, making it flexible for continually learning without human feedback. Our experimental results show that COPR outperforms strong Continuous Learning (CL) baselines when it comes to consistently aligning with human preferences on incremental tasks and domains.",
      "arxiv_id": "2310.15694v5",
      "published": "2023-10-24",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2310.15694v5",
      "pdf_url": "https://arxiv.org/pdf/2310.15694v5.pdf"
    },
    {
      "title": "Adaptive Preference Scaling for Reinforcement Learning with Human Feedback",
      "authors": [
        "Ilgee Hong",
        "Zichong Li",
        "Alexander Bukharin",
        "Yixiao Li",
        "Haoming Jiang",
        "Tianbao Yang",
        "Tuo Zhao"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is a prevalent approach to align AI systems with human values by learning rewards from human preference data. Due to various reasons, however, such data typically takes the form of rankings over pairs of trajectory segments, which fails to capture the varying strengths of preferences across different pairs. In this paper, we propose a novel adaptive preference loss, underpinned by distributionally robust optimization (DRO), designed to address this uncertainty in preference strength. By incorporating an adaptive scaling parameter into the loss for each pair, our method increases the flexibility of the reward function. Specifically, it assigns small scaling parameters to pairs with ambiguous preferences, leading to more comparable rewards, and large scaling parameters to those with clear preferences for more distinct rewards. Computationally, our proposed loss function is strictly convex and univariate with respect to each scaling parameter, enabling its efficient optimization through a simple second-order algorithm. Our method is versatile and can be readily adapted to various preference optimization frameworks, including direct preference optimization (DPO). Our experiments with robotic control and natural language generation with large language models (LLMs) show that our method not only improves policy performance but also aligns reward function selection more closely with policy optimization, simplifying the hyperparameter tuning process.",
      "arxiv_id": "2406.02764v1",
      "published": "2024-06-04",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2406.02764v1",
      "pdf_url": "https://arxiv.org/pdf/2406.02764v1.pdf"
    },
    {
      "title": "Alignment as Distribution Learning: Your Preference Model is Explicitly a Language Model",
      "authors": [
        "Jihun Yun",
        "Juno Kim",
        "Jongho Park",
        "Junhyuck Kim",
        "Jongha Jon Ryu",
        "Jaewoong Cho",
        "Kwang-Sung Jun"
      ],
      "summary": "Alignment via reinforcement learning from human feedback (RLHF) has become the dominant paradigm for controlling the quality of outputs from large language models (LLMs). However, when viewed as `loss + regularization,' the standard RLHF objective lacks theoretical justification and incentivizes degenerate, deterministic solutions, an issue that variants such as Direct Policy Optimization (DPO) also inherit. In this paper, we rethink alignment by framing it as \\emph{distribution learning} from pairwise preference feedback by explicitly modeling how information about the target language model bleeds through the preference data. This explicit modeling leads us to propose three principled learning objectives: preference maximum likelihood estimation, preference distillation, and reverse KL minimization. We theoretically show that all three approaches enjoy strong non-asymptotic $O(1/n)$ convergence to the target language model, naturally avoiding degeneracy and reward overfitting. Finally, we empirically demonstrate that our distribution learning framework, especially preference distillation, consistently outperforms or matches the performances of RLHF and DPO across various tasks and models.",
      "arxiv_id": "2506.01523v1",
      "published": "2025-06-02",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2506.01523v1",
      "pdf_url": "https://arxiv.org/pdf/2506.01523v1.pdf"
    },
    {
      "title": "Conceptual Belief-Informed Reinforcement Learning",
      "authors": [
        "Xingrui Gu",
        "Chuyi Jiang",
        "Laixi Shi"
      ],
      "summary": "Reinforcement learning (RL) has achieved significant success but is hindered by inefficiency and instability, relying on large amounts of trial-and-error data and failing to efficiently use past experiences to guide decisions. However, humans achieve remarkably efficient learning from experience, attributed to abstracting concepts and updating associated probabilistic beliefs by integrating both uncertainty and prior knowledge, as observed by cognitive science. Inspired by this, we introduce Conceptual Belief-Informed Reinforcement Learning to emulate human intelligence (HI-RL), an efficient experience utilization paradigm that can be directly integrated into existing RL frameworks. HI-RL forms concepts by extracting high-level categories of critical environmental information and then constructs adaptive concept-associated probabilistic beliefs as experience priors to guide value or policy updates. We evaluate HI-RL by integrating it into various existing value- and policy-based algorithms (DQN, PPO, SAC, and TD3) and demonstrate consistent improvements in sample efficiency and performance across both discrete and continuous control benchmarks.",
      "arxiv_id": "2410.01739v4",
      "published": "2024-10-02",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.01739v4",
      "pdf_url": "https://arxiv.org/pdf/2410.01739v4.pdf"
    },
    {
      "title": "Reinforcement Learning from Multi-level and Episodic Human Feedback",
      "authors": [
        "Muhammad Qasim Elahi",
        "Somtochukwu Oguchienti",
        "Maheed H. Ahmed",
        "Mahsa Ghasemi"
      ],
      "summary": "Designing an effective reward function has long been a challenge in reinforcement learning, particularly for complex tasks in unstructured environments. To address this, various learning paradigms have emerged that leverage different forms of human input to specify or refine the reward function. Reinforcement learning from human feedback is a prominent approach that utilizes human comparative feedback, expressed as a preference for one behavior over another, to tackle this problem. In contrast to comparative feedback, we explore multi-level human feedback, which is provided in the form of a score at the end of each episode. This type of feedback offers more coarse but informative signals about the underlying reward function than binary feedback. Additionally, it can handle non-Markovian rewards, as it is based on the evaluation of an entire episode. We propose an algorithm to efficiently learn both the reward function and the optimal policy from this form of feedback. Moreover, we show that the proposed algorithm achieves sublinear regret and demonstrate its empirical effectiveness through extensive simulations.",
      "arxiv_id": "2504.14732v3",
      "published": "2025-04-20",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.14732v3",
      "pdf_url": "https://arxiv.org/pdf/2504.14732v3.pdf"
    },
    {
      "title": "A Comprehensive Survey of Direct Preference Optimization: Datasets, Theories, Variants, and Applications",
      "authors": [
        "Wenyi Xiao",
        "Zechuan Wang",
        "Leilei Gan",
        "Shuai Zhao",
        "Zongrui Li",
        "Ruirui Lei",
        "Wanggui He",
        "Luu Anh Tuan",
        "Long Chen",
        "Hao Jiang",
        "Zhou Zhao",
        "Fei Wu"
      ],
      "summary": "With the rapid advancement of large language models (LLMs), aligning policy models with human preferences has become increasingly critical. Direct Preference Optimization (DPO) has emerged as a promising approach for alignment, acting as an RL-free alternative to Reinforcement Learning from Human Feedback (RLHF). Despite DPO's various advancements and inherent limitations, an in-depth review of these aspects is currently lacking in the literature. In this work, we present a comprehensive review of the challenges and opportunities in DPO, covering theoretical analyses, variants, relevant preference datasets, and applications. Specifically, we categorize recent studies on DPO based on key research questions to provide a thorough understanding of DPO's current landscape. Additionally, we propose several future research directions to offer insights on model alignment for the research community. An updated collection of relevant papers can be found on https://github.com/Mr-Loevan/DPO-Survey.",
      "arxiv_id": "2410.15595v3",
      "published": "2024-10-21",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.15595v3",
      "pdf_url": "https://arxiv.org/pdf/2410.15595v3.pdf"
    },
    {
      "title": "Curiosity-Driven Reinforcement Learning from Human Feedback",
      "authors": [
        "Haoran Sun",
        "Yekun Chai",
        "Shuohuan Wang",
        "Yu Sun",
        "Hua Wu",
        "Haifeng Wang"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has proven effective in aligning large language models (LLMs) with human preferences, but often at the cost of reduced output diversity. This trade-off between diversity and alignment quality remains a significant challenge. Drawing inspiration from curiosity-driven exploration in reinforcement learning, we introduce curiosity-driven RLHF (CD-RLHF), a framework that incorporates intrinsic rewards for novel states, alongside traditional sparse extrinsic rewards, to optimize both output diversity and alignment quality. We demonstrate the effectiveness of CD-RLHF through extensive experiments on a range of tasks, including text summarization and instruction following. Our approach achieves significant gains in diversity on multiple diversity-oriented metrics while maintaining alignment with human preferences comparable to standard RLHF. We make our code publicly available at https://github.com/ernie-research/CD-RLHF.",
      "arxiv_id": "2501.11463v2",
      "published": "2025-01-20",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.11463v2",
      "pdf_url": "https://arxiv.org/pdf/2501.11463v2.pdf"
    },
    {
      "title": "Mix- and MoE-DPO: A Variational Inference Approach to Direct Preference Optimization",
      "authors": [
        "Jason Bohne",
        "Pawel Polak",
        "David Rosenberg",
        "Brian Bloniarz",
        "Gary Kazantsev"
      ],
      "summary": "Direct Preference Optimization (DPO) has recently emerged as a simple and effective alternative to reinforcement learning from human feedback (RLHF) for aligning large language models (LLMs) with user preferences. However, existing DPO formulations rely on a single monolithic model, which limits their expressivity in multi-task settings and their adaptability to heterogeneous or diverse preference distributions. In this work, we propose Mix- and MoE-DPO, a framework that extends DPO with both soft mixture models and mixture-of-experts (MoE) architectures, using a stochastic variational inference approach. Our method introduces a latent-variable model over expert assignments and optimizes a variational evidence lower bound (ELBO), enabling stable and efficient learning of specialized expert policies from preference data. Mix- and MoE-DPO provides three key advantages over standard DPO: (i) generalization via universal function approximation through mixtures; (ii) reward and policy specialization through expert components tailored to distinct preference modes; and (iii) contextual alignment through input-dependent soft gating that enables user-specific mixture policies. Our framework supports both shared base architectures with expert-specific policy heads and fully independent expert models, allowing flexible trade-offs between parameter efficiency and specialization. We validate our approach on a variety of model sizes and multi-preference datasets, demonstrating that Mix- and MoE-DPO offers a powerful and scalable method for preference-based LLM alignment.",
      "arxiv_id": "2510.08256v1",
      "published": "2025-10-09",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.08256v1",
      "pdf_url": "https://arxiv.org/pdf/2510.08256v1.pdf"
    },
    {
      "title": "LiPO: Listwise Preference Optimization through Learning-to-Rank",
      "authors": [
        "Tianqi Liu",
        "Zhen Qin",
        "Junru Wu",
        "Jiaming Shen",
        "Misha Khalman",
        "Rishabh Joshi",
        "Yao Zhao",
        "Mohammad Saleh",
        "Simon Baumgartner",
        "Jialu Liu",
        "Peter J. Liu",
        "Xuanhui Wang"
      ],
      "summary": "Aligning language models (LMs) with curated human feedback is critical to control their behaviors in real-world applications. Several recent policy optimization methods, such as DPO and SLiC, serve as promising alternatives to the traditional Reinforcement Learning from Human Feedback (RLHF) approach. In practice, human feedback often comes in a format of a ranked list over multiple responses to amortize the cost of reading prompt. Multiple responses can also be ranked by reward models or AI feedback. There lacks such a thorough study on directly fitting upon a list of responses. In this work, we formulate the LM alignment as a \\textit{listwise} ranking problem and describe the LiPO framework, where the policy can potentially learn more effectively from a ranked list of plausible responses given the prompt. This view draws an explicit connection to Learning-to-Rank (LTR), where most existing preference optimization work can be mapped to existing ranking objectives. Following this connection, we provide an examination of ranking objectives that are not well studied for LM alignment with DPO and SLiC as special cases when list size is two. In particular, we highlight a specific method, LiPO-$\u03bb$, which leverages a state-of-the-art \\textit{listwise} ranking objective and weights each preference pair in a more advanced manner. We show that LiPO-$\u03bb$ can outperform DPO variants and SLiC by a clear margin on several preference alignment tasks with both curated and real rankwise preference data.",
      "arxiv_id": "2402.01878v3",
      "published": "2024-02-02",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2402.01878v3",
      "pdf_url": "https://arxiv.org/pdf/2402.01878v3.pdf"
    },
    {
      "title": "Pre-DPO: Improving Data Utilization in Direct Preference Optimization Using a Guiding Reference Model",
      "authors": [
        "Junshu Pan",
        "Wei Shen",
        "Shulin Huang",
        "Qiji Zhou",
        "Yue Zhang"
      ],
      "summary": "Direct Preference Optimization (DPO) simplifies reinforcement learning from human feedback (RLHF) for large language models (LLMs) by directly optimizing human preferences without an explicit reward model. We find that during DPO training, the reference model plays the role of a data weight adjuster. However, the common practice of initializing the policy and reference models identically in DPO can lead to inefficient data utilization and impose a performance ceiling. Meanwhile, the lack of a reference model in Simple Preference Optimization (SimPO) reduces training robustness and necessitates stricter conditions to prevent catastrophic forgetting. In this work, we propose Pre-DPO, a simple yet effective DPO-based training paradigm that enhances preference optimization performance by leveraging a guiding reference model. This reference model provides foresight into the optimal policy state achievable through the training preference data, serving as a guiding mechanism that adaptively assigns higher weights to samples more suitable for the model and lower weights to those less suitable. Extensive experiments on AlpacaEval 2.0 and Arena-Hard v0.1 benchmarks demonstrate that Pre-DPO consistently improves the performance of both DPO and SimPO, without relying on external models or additional data.",
      "arxiv_id": "2504.15843v2",
      "published": "2025-04-22",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.15843v2",
      "pdf_url": "https://arxiv.org/pdf/2504.15843v2.pdf"
    },
    {
      "title": "Interactive Groupwise Comparison for Reinforcement Learning from Human Feedback",
      "authors": [
        "Jan Kompatscher",
        "Danqing Shi",
        "Giovanna Varni",
        "Tino Weinkauf",
        "Antti Oulasvirta"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has emerged as a key enabling technology for aligning AI behavior with human preferences. The traditional way to collect data in RLHF is via pairwise comparisons: human raters are asked to indicate which one of two samples they prefer. We present an interactive visualization that better exploits the human visual ability to compare and explore whole groups of samples. The interface is comprised of two linked views: 1) an exploration view showing a contextual overview of all sampled behaviors organized in a hierarchical clustering structure; and 2) a comparison view displaying two selected groups of behaviors for user queries. Users can efficiently explore large sets of behaviors by iterating between these two views. Additionally, we devised an active learning approach suggesting groups for comparison. As shown by our evaluation in six simulated robotics tasks, our approach increases the final policy returns by 69.34%. It leads to lower error rates and better policies. We open-source the code that can be easily integrated into the RLHF training loop, supporting research on human-AI alignment.",
      "arxiv_id": "2507.04340v1",
      "published": "2025-07-06",
      "categories": [
        "cs.LG",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2507.04340v1",
      "pdf_url": "https://arxiv.org/pdf/2507.04340v1.pdf"
    },
    {
      "title": "Mapping Neural Signals to Agent Performance, A Step Towards Reinforcement Learning from Neural Feedback",
      "authors": [
        "Julia Santaniello",
        "Matthew Russell",
        "Benson Jiang",
        "Donatello Sassaroli",
        "Robert Jacob",
        "Jivko Sinapov"
      ],
      "summary": "Implicit Human-in-the-Loop Reinforcement Learning (HITL-RL) is a methodology that integrates passive human feedback into autonomous agent training while minimizing human workload. However, existing methods often rely on active instruction, requiring participants to teach an agent through unnatural expression or gesture. We introduce NEURO-LOOP, an implicit feedback framework that utilizes the intrinsic human reward system to drive human-agent interaction. This work demonstrates the feasibility of a critical first step in the NEURO-LOOP framework: mapping brain signals to agent performance. Using functional near-infrared spectroscopy (fNIRS), we design a dataset to enable future research using passive Brain-Computer Interfaces for Human-in-the-Loop Reinforcement Learning. Participants are instructed to observe or guide a reinforcement learning agent in its environment while signals from the prefrontal cortex are collected. We conclude that a relationship between fNIRS data and agent performance exists using classical machine learning techniques. Finally, we highlight the potential that neural interfaces may offer to future applications of human-agent interaction, assistive AI, and adaptive autonomous systems.",
      "arxiv_id": "2506.12636v1",
      "published": "2025-06-14",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2506.12636v1",
      "pdf_url": "https://arxiv.org/pdf/2506.12636v1.pdf"
    },
    {
      "title": "Improving Multimodal Interactive Agents with Reinforcement Learning from Human Feedback",
      "authors": [
        "Josh Abramson",
        "Arun Ahuja",
        "Federico Carnevale",
        "Petko Georgiev",
        "Alex Goldin",
        "Alden Hung",
        "Jessica Landon",
        "Jirka Lhotka",
        "Timothy Lillicrap",
        "Alistair Muldal",
        "George Powell",
        "Adam Santoro",
        "Guy Scully",
        "Sanjana Srivastava",
        "Tamara von Glehn",
        "Greg Wayne",
        "Nathaniel Wong",
        "Chen Yan",
        "Rui Zhu"
      ],
      "summary": "An important goal in artificial intelligence is to create agents that can both interact naturally with humans and learn from their feedback. Here we demonstrate how to use reinforcement learning from human feedback (RLHF) to improve upon simulated, embodied agents trained to a base level of competency with imitation learning. First, we collected data of humans interacting with agents in a simulated 3D world. We then asked annotators to record moments where they believed that agents either progressed toward or regressed from their human-instructed goal. Using this annotation data we leveraged a novel method - which we call \"Inter-temporal Bradley-Terry\" (IBT) modelling - to build a reward model that captures human judgments. Agents trained to optimise rewards delivered from IBT reward models improved with respect to all of our metrics, including subsequent human judgment during live interactions with agents. Altogether our results demonstrate how one can successfully leverage human judgments to improve agent behaviour, allowing us to use reinforcement learning in complex, embodied domains without programmatic reward functions. Videos of agent behaviour may be found at https://youtu.be/v_Z9F2_eKk4.",
      "arxiv_id": "2211.11602v1",
      "published": "2022-11-21",
      "categories": [
        "cs.LG",
        "cs.HC",
        "cs.MA"
      ],
      "url": "https://arxiv.org/abs/2211.11602v1",
      "pdf_url": "https://arxiv.org/pdf/2211.11602v1.pdf"
    },
    {
      "title": "Robust Reinforcement Learning from Human Feedback for Large Language Models Fine-Tuning",
      "authors": [
        "Kai Ye",
        "Hongyi Zhou",
        "Jin Zhu",
        "Francesco Quinzan",
        "Chengchun Shi"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has emerged as a key technique for aligning the output of large language models (LLMs) with human preferences. To learn the reward function, most existing RLHF algorithms use the Bradley-Terry model, which relies on assumptions about human preferences that may not reflect the complexity and variability of real-world judgments. In this paper, we propose a robust algorithm to enhance the performance of existing approaches under such reward model misspecifications. Theoretically, our algorithm reduces the variance of reward and policy estimators, leading to improved regret bounds. Empirical evaluations on LLM benchmark datasets demonstrate that the proposed algorithm consistently outperforms existing methods, with 77-81% of responses being favored over baselines on the Anthropic Helpful and Harmless dataset.",
      "arxiv_id": "2504.03784v4",
      "published": "2025-04-03",
      "categories": [
        "stat.ML",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.03784v4",
      "pdf_url": "https://arxiv.org/pdf/2504.03784v4.pdf"
    },
    {
      "title": "DIPPER: Direct Preference Optimization to Accelerate Primitive-Enabled Hierarchical Reinforcement Learning",
      "authors": [
        "Utsav Singh",
        "Souradip Chakraborty",
        "Wesley A. Suttle",
        "Brian M. Sadler",
        "Vinay P Namboodiri",
        "Amrit Singh Bedi"
      ],
      "summary": "Learning control policies to perform complex robotics tasks from human preference data presents significant challenges. On the one hand, the complexity of such tasks typically requires learning policies to perform a variety of subtasks, then combining them to achieve the overall goal. At the same time, comprehensive, well-engineered reward functions are typically unavailable in such problems, while limited human preference data often is; making efficient use of such data to guide learning is therefore essential. Methods for learning to perform complex robotics tasks from human preference data must overcome both these challenges simultaneously. In this work, we introduce DIPPER: Direct Preference Optimization to Accelerate Primitive-Enabled Hierarchical Reinforcement Learning, an efficient hierarchical approach that leverages direct preference optimization to learn a higher-level policy and reinforcement learning to learn a lower-level policy. DIPPER enjoys improved computational efficiency due to its use of direct preference optimization instead of standard preference-based approaches such as reinforcement learning from human feedback, while it also mitigates the well-known hierarchical reinforcement learning issues of non-stationarity and infeasible subgoal generation due to our use of primitive-informed regularization inspired by a novel bi-level optimization formulation of the hierarchical reinforcement learning problem. To validate our approach, we perform extensive experimental analysis on a variety of challenging robotics tasks, demonstrating that DIPPER outperforms hierarchical and non-hierarchical baselines, while ameliorating the non-stationarity and infeasible subgoal generation issues of hierarchical reinforcement learning.",
      "arxiv_id": "2406.10892v3",
      "published": "2024-06-16",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2406.10892v3",
      "pdf_url": "https://arxiv.org/pdf/2406.10892v3.pdf"
    },
    {
      "title": "Sequence to Sequence Reward Modeling: Improving RLHF by Language Feedback",
      "authors": [
        "Jiayi Zhou",
        "Jiaming Ji",
        "Juntao Dai",
        "Yaodong Yang"
      ],
      "summary": "Aligning the behavior of Large language models (LLMs) with human intentions and values remains a critical challenge. Reinforcement learning from human feedback (RLHF) aligns LLMs by training a reward model (RM) on human preferences and fine-tuning the LLMs to maximize RM feedback. Despite its effectiveness and popularity, RLHF is prone to biased local optimization. It means RM fails to provide feedback that accurately aligns with human preference, causing LLMs to explore unexpected generalizations, and failing to achieve alignment objectives. To mitigate this issue, we propose a novel \\textit{sequence-to-sequence (seq2seq) reward modeling} method. Its key insight is that learning from language feedback rather than scalar feedback improves RLHF without additional annotations. We replaced the reward modeling target from binary maximum likelihood estimation (MLE) with sequence MLE. This method enables richer and fine-grained language feedback without additional annotations, models, or training stages. Our experiments demonstrated its effectiveness, specifically, reducing the refusal-to-response paradigm in single-turn safety dialogues and the long-response bias in text summarization tasks. We provide further analysis that seq2seq RM improves RLHF performance across 2B and 7B LLMs on 3 NLP tasks, achieving an average win rate of 76.9\\%. We further show that seq2seq RM can still improve the performance of RLHF under out-of-distribution prompts.",
      "arxiv_id": "2409.00162v1",
      "published": "2024-08-30",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2409.00162v1",
      "pdf_url": "https://arxiv.org/pdf/2409.00162v1.pdf"
    },
    {
      "title": "Efficient Reinforcement Learning from Human Feedback via Bayesian Preference Inference",
      "authors": [
        "Matteo Cercola",
        "Valeria Capretti",
        "Simone Formentin"
      ],
      "summary": "Learning from human preferences is a cornerstone of aligning machine learning models with subjective human judgments. Yet, collecting such preference data is often costly and time-consuming, motivating the need for more efficient learning paradigms. Two established approaches offer complementary advantages: RLHF scales effectively to high-dimensional tasks such as LLM fine-tuning, while PBO achieves greater sample efficiency through active querying. We propose a hybrid framework that unifies RLHF's scalability with PBO's query efficiency by integrating an acquisition-driven module into the RLHF pipeline, thereby enabling active and sample-efficient preference gathering. We validate the proposed approach on two representative domains: (i) high-dimensional preference optimization and (ii) LLM fine-tuning. Experimental results demonstrate consistent improvements in both sample efficiency and overall performance across these tasks.",
      "arxiv_id": "2511.04286v1",
      "published": "2025-11-06",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2511.04286v1",
      "pdf_url": "https://arxiv.org/pdf/2511.04286v1.pdf"
    },
    {
      "title": "The Alignment Ceiling: Objective Mismatch in Reinforcement Learning from Human Feedback",
      "authors": [
        "Nathan Lambert",
        "Roberto Calandra"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has emerged as a powerful technique to make large language models (LLMs) more capable in complex settings. RLHF proceeds as collecting human preference data, training a reward model on said data, and optimizing a base ML model with respect to said reward for extrinsic evaluation metrics (e.g. MMLU, GSM8k). RLHF relies on many assumptions about how the various pieces fit together, such as a reward model capturing human preferences and an RL optimizer extracting the right signal from a reward model. As the RLHF process involves many distinct design decisions, it is easy to assume that multiple processes are correlated and therefore numerically linked. This apparent correlation is often not true, where reward models are easily overoptimized or RL optimizers can reduce performance on tasks not modeled in the data. Notable manifestations of models trained with imperfect RLHF systems are those that are prone to refusing basic requests for safety reasons or appearing lazy in generations. As chat model evaluation becomes increasingly nuanced, the reliance on a perceived link between reward model training, RL scores, and downstream performance drives these issues, which we describe as an objective mismatch. In this paper, we illustrate the causes of this issue, reviewing relevant literature from model-based reinforcement learning, and argue for solutions. By solving objective mismatch in RLHF, the ML models of the future will be more precisely aligned to user instructions for both safety and helpfulness.",
      "arxiv_id": "2311.00168v2",
      "published": "2023-10-31",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2311.00168v2",
      "pdf_url": "https://arxiv.org/pdf/2311.00168v2.pdf"
    },
    {
      "title": "Oracle-RLAIF: An Improved Fine-Tuning Framework for Multi-modal Video Models through Reinforcement Learning from Ranking Feedback",
      "authors": [
        "Derek Shi",
        "Ruben Glatt",
        "Christine Klymko",
        "Shubham Mohole",
        "Hongjun Choi",
        "Shashank Kushwaha",
        "Sam Sakla",
        "Felipe Leno da Silva"
      ],
      "summary": "Recent advances in large video-language models (VLMs) rely on extensive fine-tuning techniques that strengthen alignment between textual and visual comprehension. Leading pipelines typically pair supervised fine-tuning (SFT) with reinforcement learning from preference data to enhance video comprehension. However, as VLMs scale in parameter size, so does the cost of gathering enough human feedback. To make fine-tuning more cost-effective, recent frameworks explore reinforcement learning with AI feedback (RLAIF), which replace human preference with AI as a judge. Current RLAIF frameworks rely on a specialized reward model trained with video narratives to create calibrated scalar rewards -- an expensive and restrictive pipeline. We propose Oracle-RLAIF, a novel framework that replaces the trained reward model with a more general Oracle ranker which acts as a drop-in model ranking candidate model responses rather than scoring them. Alongside Oracle-RLAIF, we introduce $GRPO_{rank}$, a novel rank-based loss function based on Group Relative Policy Optimization (GRPO) that directly optimizes ordinal feedback with rank-aware advantages. Empirically, we demonstrate that Oracle-RLAIF consistently outperforms leading VLMs using existing fine-tuning methods when evaluated across various video comprehension benchmarks. Oracle-RLAIF paves the path to creating flexible and data-efficient frameworks for aligning large multi-modal video models with reinforcement learning from rank rather than score.",
      "arxiv_id": "2510.02561v1",
      "published": "2025-10-02",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.02561v1",
      "pdf_url": "https://arxiv.org/pdf/2510.02561v1.pdf"
    },
    {
      "title": "Unveiling the Role of Expert Guidance: A Comparative Analysis of User-centered Imitation Learning and Traditional Reinforcement Learning",
      "authors": [
        "Amr Gomaa",
        "Bilal Mahdy"
      ],
      "summary": "Integration of human feedback plays a key role in improving the learning capabilities of intelligent systems. This comparative study delves into the performance, robustness, and limitations of imitation learning compared to traditional reinforcement learning methods within these systems. Recognizing the value of human-in-the-loop feedback, we investigate the influence of expert guidance and suboptimal demonstrations on the learning process. Through extensive experimentation and evaluations conducted in a pre-existing simulation environment using the Unity platform, we meticulously analyze the effectiveness and limitations of these learning approaches. The insights gained from this study contribute to the advancement of human-centered artificial intelligence by highlighting the benefits and challenges associated with the incorporation of human feedback into the learning process. Ultimately, this research promotes the development of models that can effectively address complex real-world problems.",
      "arxiv_id": "2410.21403v1",
      "published": "2024-10-28",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2410.21403v1",
      "pdf_url": "https://arxiv.org/pdf/2410.21403v1.pdf"
    },
    {
      "title": "C2-DPO: Constrained Controlled Direct Preference Optimization",
      "authors": [
        "Kavosh Asadi",
        "Julien Han",
        "Idan Pipano",
        "Xingzi Xu",
        "Dominique Perrault-Joncas",
        "Shoham Sabach",
        "Karim Bouyarmane",
        "Mohammad Ghavamzadeh"
      ],
      "summary": "Direct preference optimization (\\texttt{DPO}) has emerged as a promising approach for solving the alignment problem in AI. In this paper, we make two counter-intuitive observations about \\texttt{DPO}. First, we show that \\texttt{DPO} loss could be derived by starting from an alternative optimization problem that only defines the KL guardrail on in-sample responses, unlike the original RLHF problem where guardrails are defined on the entire distribution. Second, we prove a surprising property of this alternative optimization problem, namely that under its optimal policy, both preferred and rejected responses tend to decrease in probability, a phenomenon typically displayed by DPO in practice. To control this behavior, we propose a set of constraints designed to limit the displacement of probability mass between the preferred and rejected responses in the reference and target policies. The resulting algorithm, which we call Constrained Controlled DPO (\\texttt{C2-DPO}), has a meaningful RLHF interpretation. By hedging against the displacement, \\texttt{C2-DPO} provides practical improvements over vanilla \\texttt{DPO} when aligning several language models using standard preference datasets.",
      "arxiv_id": "2502.17507v2",
      "published": "2025-02-22",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2502.17507v2",
      "pdf_url": "https://arxiv.org/pdf/2502.17507v2.pdf"
    },
    {
      "title": "Nash Learning from Human Feedback",
      "authors": [
        "R\u00e9mi Munos",
        "Michal Valko",
        "Daniele Calandriello",
        "Mohammad Gheshlaghi Azar",
        "Mark Rowland",
        "Zhaohan Daniel Guo",
        "Yunhao Tang",
        "Matthieu Geist",
        "Thomas Mesnard",
        "Andrea Michi",
        "Marco Selvi",
        "Sertan Girgin",
        "Nikola Momchev",
        "Olivier Bachem",
        "Daniel J. Mankowitz",
        "Doina Precup",
        "Bilal Piot"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has emerged as the main paradigm for aligning large language models (LLMs) with human preferences. Typically, RLHF involves the initial step of learning a reward model from human feedback, often expressed as preferences between pairs of text generations produced by a pre-trained LLM. Subsequently, the LLM's policy is fine-tuned by optimizing it to maximize the reward model through a reinforcement learning algorithm. However, an inherent limitation of current reward models is their inability to fully represent the richness of human preferences and their dependency on the sampling distribution.   In this study, we introduce an alternative pipeline for the fine-tuning of LLMs using pairwise human feedback. Our approach entails the initial learning of a preference model, which is conditioned on two inputs given a prompt, followed by the pursuit of a policy that consistently generates responses preferred over those generated by any competing policy, thus defining the Nash equilibrium of this preference model. We term this approach Nash learning from human feedback (NLHF).   In the context of a tabular policy representation, we present a novel algorithmic solution, Nash-MD, founded on the principles of mirror descent. This algorithm produces a sequence of policies, with the last iteration converging to the regularized Nash equilibrium. Additionally, we explore parametric representations of policies and introduce gradient descent algorithms for deep-learning architectures. To demonstrate the effectiveness of our approach, we present experimental results involving the fine-tuning of a LLM for a text summarization task. We believe NLHF offers a compelling avenue for preference learning and policy optimization with the potential of advancing the field of aligning LLMs with human preferences.",
      "arxiv_id": "2312.00886v4",
      "published": "2023-12-01",
      "categories": [
        "stat.ML",
        "cs.AI",
        "cs.GT",
        "cs.LG",
        "cs.MA"
      ],
      "url": "https://arxiv.org/abs/2312.00886v4",
      "pdf_url": "https://arxiv.org/pdf/2312.00886v4.pdf"
    },
    {
      "title": "When Your AIs Deceive You: Challenges of Partial Observability in Reinforcement Learning from Human Feedback",
      "authors": [
        "Leon Lang",
        "Davis Foote",
        "Stuart Russell",
        "Anca Dragan",
        "Erik Jenner",
        "Scott Emmons"
      ],
      "summary": "Past analyses of reinforcement learning from human feedback (RLHF) assume that the human evaluators fully observe the environment. What happens when human feedback is based only on partial observations? We formally define two failure cases: deceptive inflation and overjustification. Modeling the human as Boltzmann-rational w.r.t. a belief over trajectories, we prove conditions under which RLHF is guaranteed to result in policies that deceptively inflate their performance, overjustify their behavior to make an impression, or both. Under the new assumption that the human's partial observability is known and accounted for, we then analyze how much information the feedback process provides about the return function. We show that sometimes, the human's feedback determines the return function uniquely up to an additive constant, but in other realistic cases, there is irreducible ambiguity. We propose exploratory research directions to help tackle these challenges, experimentally validate both the theoretical concerns and potential mitigations, and caution against blindly applying RLHF in partially observable settings.",
      "arxiv_id": "2402.17747v5",
      "published": "2024-02-27",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2402.17747v5",
      "pdf_url": "https://arxiv.org/pdf/2402.17747v5.pdf"
    },
    {
      "title": "SEE-DPO: Self Entropy Enhanced Direct Preference Optimization",
      "authors": [
        "Shivanshu Shekhar",
        "Shreyas Singh",
        "Tong Zhang"
      ],
      "summary": "Direct Preference Optimization (DPO) has been successfully used to align large language models (LLMs) according to human preferences, and more recently it has also been applied to improving the quality of text-to-image diffusion models. However, DPO-based methods such as SPO, Diffusion-DPO, and D3PO are highly susceptible to overfitting and reward hacking, especially when the generative model is optimized to fit out-of-distribution during prolonged training. To overcome these challenges and stabilize the training of diffusion models, we introduce a self-entropy regularization mechanism in reinforcement learning from human feedback. This enhancement improves DPO training by encouraging broader exploration and greater robustness. Our regularization technique effectively mitigates reward hacking, leading to improved stability and enhanced image quality across the latent space. Extensive experiments demonstrate that integrating human feedback with self-entropy regularization can significantly boost image diversity and specificity, achieving state-of-the-art results on key image generation metrics.",
      "arxiv_id": "2411.04712v2",
      "published": "2024-11-06",
      "categories": [
        "cs.CV",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2411.04712v2",
      "pdf_url": "https://arxiv.org/pdf/2411.04712v2.pdf"
    },
    {
      "title": "Active Preference Optimization for Sample Efficient RLHF",
      "authors": [
        "Nirjhar Das",
        "Souradip Chakraborty",
        "Aldo Pacchiano",
        "Sayak Ray Chowdhury"
      ],
      "summary": "Large Language Models (LLMs) aligned using Reinforcement Learning from Human Feedback (RLHF) have shown remarkable generation abilities in numerous tasks. However, collecting high-quality human preferences creates costly bottlenecks in practical deployments, and hence, training data are often budgeted. In these scenarios, it is crucial to collect training data (e.g., contexts, a pair of generations for each context, and a preference indicating which generation is better) carefully, yet most of the existing methods sample contexts uniformly at random from a given collection. Given this, under the Bradley-Terry-Luce preference model and with a small budget of training data, we show that uniform sampling of contexts could lead to a policy (i.e., an aligned model) that suffers a constant sub-optimality gap from the optimal policy. This highlights the need for an adaptive context sampling strategy for effective alignment under a small sample budget. To address this, we reformulate RLHF within the contextual preference bandit framework, treating generations as actions, and give a nearly complete characterization of the sub-optimality gap in terms of both lower and upper bounds. First, when the action set is a $d$-dimensional hypercube and the number of samples is $T$, we show an $\u03a9(d/\\sqrt{T})$ lower bound. Next, we propose an algorithm, $\\textit{Active Preference Optimization}$ ($\\texttt{APO}$), that iteratively collects preferences for the most uncertain contexts. We show that the sub-optimality gap of the policy learned via $\\texttt{APO}$ matches the lower bound up to a log factor and a non-linearity constant. Finally, we perform experiments on practical datasets to validate $\\texttt{APO}$'s efficacy over existing methods, establishing it as a sample-efficient and cost-effective solution for LLM alignment.",
      "arxiv_id": "2402.10500v3",
      "published": "2024-02-16",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2402.10500v3",
      "pdf_url": "https://arxiv.org/pdf/2402.10500v3.pdf"
    },
    {
      "title": "WPO: Enhancing RLHF with Weighted Preference Optimization",
      "authors": [
        "Wenxuan Zhou",
        "Ravi Agrawal",
        "Shujian Zhang",
        "Sathish Reddy Indurthi",
        "Sanqiang Zhao",
        "Kaiqiang Song",
        "Silei Xu",
        "Chenguang Zhu"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is a promising solution to align large language models (LLMs) more closely with human values. Off-policy preference optimization, where the preference data is obtained from other models, is widely adopted due to its cost efficiency and scalability. However, off-policy preference optimization often suffers from a distributional gap between the policy used for data collection and the target policy, leading to suboptimal optimization. In this paper, we propose a novel strategy to mitigate this problem by simulating on-policy learning with off-policy preference data. Our Weighted Preference Optimization (WPO) method adapts off-policy data to resemble on-policy data more closely by reweighting preference pairs according to their probability under the current policy. This method not only addresses the distributional gap problem but also enhances the optimization process without incurring additional costs. We validate our method on instruction following benchmarks including Alpaca Eval 2 and MT-bench. WPO not only outperforms Direct Preference Optimization (DPO) by up to 5.6% on Alpaca Eval 2 but also establishes a remarkable length-controlled winning rate against GPT-4-turbo of 76.7% based on Gemma-2-9b-it. We release the code and models at https://github.com/wzhouad/WPO.",
      "arxiv_id": "2406.11827v2",
      "published": "2024-06-17",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2406.11827v2",
      "pdf_url": "https://arxiv.org/pdf/2406.11827v2.pdf"
    },
    {
      "title": "Is Feedback All You Need? Leveraging Natural Language Feedback in Goal-Conditioned Reinforcement Learning",
      "authors": [
        "Sabrina McCallum",
        "Max Taylor-Davies",
        "Stefano V. Albrecht",
        "Alessandro Suglia"
      ],
      "summary": "Despite numerous successes, the field of reinforcement learning (RL) remains far from matching the impressive generalisation power of human behaviour learning. One possible way to help bridge this gap be to provide RL agents with richer, more human-like feedback expressed in natural language. To investigate this idea, we first extend BabyAI to automatically generate language feedback from the environment dynamics and goal condition success. Then, we modify the Decision Transformer architecture to take advantage of this additional signal. We find that training with language feedback either in place of or in addition to the return-to-go or goal descriptions improves agents' generalisation performance, and that agents can benefit from feedback even when this is only available during training, but not at inference.",
      "arxiv_id": "2312.04736v1",
      "published": "2023-12-07",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2312.04736v1",
      "pdf_url": "https://arxiv.org/pdf/2312.04736v1.pdf"
    },
    {
      "title": "The Power of Active Multi-Task Learning in Reinforcement Learning from Human Feedback",
      "authors": [
        "Ruitao Chen",
        "Liwei Wang"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has contributed to performance improvements in large language models. To tackle its reliance on substantial amounts of human-labeled data, a successful approach is multi-task representation learning, which involves learning a high-quality, low-dimensional representation from a wide range of source tasks. In this paper, we formulate RLHF as the contextual dueling bandit problem and assume a common linear representation. We demonstrate that the sample complexity of source tasks in multi-task RLHF can be reduced by considering task relevance and allocating different sample sizes to source tasks with varying task relevance. We further propose an algorithm to estimate task relevance by a small number of additional data and then learn a policy. We prove that to achieve $\\varepsilon-$optimal, the sample complexity of the source tasks can be significantly reduced compared to uniform sampling. Additionally, the sample complexity of the target task is only linear in the dimension of the latent space, thanks to representation learning.",
      "arxiv_id": "2405.11226v2",
      "published": "2024-05-18",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.11226v2",
      "pdf_url": "https://arxiv.org/pdf/2405.11226v2.pdf"
    },
    {
      "title": "Neural Dueling Bandits: Preference-Based Optimization with Human Feedback",
      "authors": [
        "Arun Verma",
        "Zhongxiang Dai",
        "Xiaoqiang Lin",
        "Patrick Jaillet",
        "Bryan Kian Hsiang Low"
      ],
      "summary": "Contextual dueling bandit is used to model the bandit problems, where a learner's goal is to find the best arm for a given context using observed noisy human preference feedback over the selected arms for the past contexts. However, existing algorithms assume the reward function is linear, which can be complex and non-linear in many real-life applications like online recommendations or ranking web search results. To overcome this challenge, we use a neural network to estimate the reward function using preference feedback for the previously selected arms. We propose upper confidence bound- and Thompson sampling-based algorithms with sub-linear regret guarantees that efficiently select arms in each round. We also extend our theoretical results to contextual bandit problems with binary feedback, which is in itself a non-trivial contribution. Experimental results on the problem instances derived from synthetic datasets corroborate our theoretical results.",
      "arxiv_id": "2407.17112v2",
      "published": "2024-07-24",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2407.17112v2",
      "pdf_url": "https://arxiv.org/pdf/2407.17112v2.pdf"
    },
    {
      "title": "Enhancing Paraphrase Type Generation: The Impact of DPO and RLHF Evaluated with Human-Ranked Data",
      "authors": [
        "Christopher Lee L\u00fcbbers"
      ],
      "summary": "Paraphrasing re-expresses meaning to enhance applications like text simplification, machine translation, and question-answering. Specific paraphrase types facilitate accurate semantic analysis and robust language models. However, existing paraphrase-type generation methods often misalign with human preferences due to reliance on automated metrics and limited human-annotated training data, obscuring crucial aspects of semantic fidelity and linguistic transformations.   This study addresses this gap by leveraging a human-ranked paraphrase-type dataset and integrating Direct Preference Optimization (DPO) to align model outputs directly with human judgments. DPO-based training increases paraphrase-type generation accuracy by 3 percentage points over a supervised baseline and raises human preference ratings by 7 percentage points. A newly created human-annotated dataset supports more rigorous future evaluations. Additionally, a paraphrase-type detection model achieves F1 scores of 0.91 for addition/deletion, 0.78 for same polarity substitution, and 0.70 for punctuation changes.   These findings demonstrate that preference data and DPO training produce more reliable, semantically accurate paraphrases, enabling downstream applications such as improved summarization and more robust question-answering. The PTD model surpasses automated metrics and provides a more reliable framework for evaluating paraphrase quality, advancing paraphrase-type research toward richer, user-aligned language generation and establishing a stronger foundation for future evaluations grounded in human-centric criteria.",
      "arxiv_id": "2506.02018v1",
      "published": "2025-05-28",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.02018v1",
      "pdf_url": "https://arxiv.org/pdf/2506.02018v1.pdf"
    },
    {
      "title": "REBEL: Reward Regularization-Based Approach for Robotic Reinforcement Learning from Human Feedback",
      "authors": [
        "Souradip Chakraborty",
        "Anukriti Singh",
        "Amisha Bhaskar",
        "Pratap Tokekar",
        "Dinesh Manocha",
        "Amrit Singh Bedi"
      ],
      "summary": "The effectiveness of reinforcement learning (RL) agents in continuous control robotics tasks is mainly dependent on the design of the underlying reward function, which is highly prone to reward hacking. A misalignment between the reward function and underlying human preferences (values, social norms) can lead to catastrophic outcomes in the real world especially in the context of robotics for critical decision making. Recent methods aim to mitigate misalignment by learning reward functions from human preferences and subsequently performing policy optimization. However, these methods inadvertently introduce a distribution shift during reward learning due to ignoring the dependence of agent-generated trajectories on the reward learning objective, ultimately resulting in sub-optimal alignment. Hence, in this work, we address this challenge by advocating for the adoption of regularized reward functions that more accurately mirror the intended behaviors of the agent. We propose a novel concept of reward regularization within the robotic RLHF (RL from Human Feedback) framework, which we refer to as \\emph{agent preferences}. Our approach uniquely incorporates not just human feedback in the form of preferences but also considers the preferences of the RL agent itself during the reward function learning process. This dual consideration significantly mitigates the issue of distribution shift in RLHF with a computationally tractable algorithm. We provide a theoretical justification for the proposed algorithm by formulating the robotic RLHF problem as a bilevel optimization problem and developing a computationally tractable version of the same. We demonstrate the efficiency of our algorithm {\\ours} in several continuous control benchmarks in DeepMind Control Suite \\cite{tassa2018deepmind}.",
      "arxiv_id": "2312.14436v3",
      "published": "2023-12-22",
      "categories": [
        "cs.RO",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2312.14436v3",
      "pdf_url": "https://arxiv.org/pdf/2312.14436v3.pdf"
    },
    {
      "title": "VickreyFeedback: Cost-efficient Data Construction for Reinforcement Learning from Human Feedback",
      "authors": [
        "Guoxi Zhang",
        "Jiuding Duan"
      ],
      "summary": "This paper addresses the cost-efficiency aspect of Reinforcement Learning from Human Feedback (RLHF). RLHF leverages datasets of human preferences over outputs of large language models (LLM)s to instill human expectations into LLMs. Although preference annotation comes with a monetized cost, the economic utility of a preference dataset has not been considered by far. What exacerbates this situation is that, given complex intransitive or cyclic relationships in preference datasets, existing algorithms for fine-tuning LLMs are still far from capturing comprehensive preferences. This raises severe cost-efficiency concerns in production environments, where preference data accumulate over time. In this paper, we discuss the fine-tuning of LLMs as a monetized economy and introduce an auction mechanism to improve the efficiency of preference data collection in dollar terms. We show that introducing an auction mechanism can play an essential role in enhancing the cost-efficiency of RLHF, while maintaining satisfactory model performance. Experimental results demonstrate that our proposed auction-based protocol is cost-effective for fine-tuning LLMs concentrating on high-quality feedback.",
      "arxiv_id": "2409.18417v2",
      "published": "2024-09-27",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.GT",
        "econ.GN"
      ],
      "url": "https://arxiv.org/abs/2409.18417v2",
      "pdf_url": "https://arxiv.org/pdf/2409.18417v2.pdf"
    },
    {
      "title": "Influencing Humans to Conform to Preference Models for RLHF",
      "authors": [
        "Stephane Hatgis-Kessell",
        "W. Bradley Knox",
        "Serena Booth",
        "Scott Niekum",
        "Peter Stone"
      ],
      "summary": "Designing a reinforcement learning from human feedback (RLHF) algorithm to approximate a human's unobservable reward function requires assuming, implicitly or explicitly, a model of human preferences. A preference model that poorly describes how humans generate preferences risks learning a poor approximation of the human's reward function. In this paper, we conduct three human studies to asses whether one can influence the expression of real human preferences to more closely conform to a desired preference model. Importantly, our approach does not seek to alter the human's unobserved reward function. Rather, we change how humans use this reward function to generate preferences, such that they better match whatever preference model is assumed by a particular RLHF algorithm. We introduce three interventions: showing humans the quantities that underlie a preference model, which is normally unobservable information derived from the reward function; training people to follow a specific preference model; and modifying the preference elicitation question. All intervention types show significant effects, providing practical tools to improve preference data quality and the resultant alignment of the learned reward functions. Overall we establish a novel research direction in model alignment: designing interfaces and training interventions to increase human conformance with the modeling assumptions of the algorithm that will learn from their input.",
      "arxiv_id": "2501.06416v2",
      "published": "2025-01-11",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2501.06416v2",
      "pdf_url": "https://arxiv.org/pdf/2501.06416v2.pdf"
    },
    {
      "title": "DIP-RL: Demonstration-Inferred Preference Learning in Minecraft",
      "authors": [
        "Ellen Novoseller",
        "Vinicius G. Goecks",
        "David Watkins",
        "Josh Miller",
        "Nicholas Waytowich"
      ],
      "summary": "In machine learning for sequential decision-making, an algorithmic agent learns to interact with an environment while receiving feedback in the form of a reward signal. However, in many unstructured real-world settings, such a reward signal is unknown and humans cannot reliably craft a reward signal that correctly captures desired behavior. To solve tasks in such unstructured and open-ended environments, we present Demonstration-Inferred Preference Reinforcement Learning (DIP-RL), an algorithm that leverages human demonstrations in three distinct ways, including training an autoencoder, seeding reinforcement learning (RL) training batches with demonstration data, and inferring preferences over behaviors to learn a reward function to guide RL. We evaluate DIP-RL in a tree-chopping task in Minecraft. Results suggest that the method can guide an RL agent to learn a reward function that reflects human preferences and that DIP-RL performs competitively relative to baselines. DIP-RL is inspired by our previous work on combining demonstrations and pairwise preferences in Minecraft, which was awarded a research prize at the 2022 NeurIPS MineRL BASALT competition, Learning from Human Feedback in Minecraft. Example trajectory rollouts of DIP-RL and baselines are located at https://sites.google.com/view/dip-rl.",
      "arxiv_id": "2307.12158v1",
      "published": "2023-07-22",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2307.12158v1",
      "pdf_url": "https://arxiv.org/pdf/2307.12158v1.pdf"
    },
    {
      "title": "Earlier Tokens Contribute More: Learning Direct Preference Optimization From Temporal Decay Perspective",
      "authors": [
        "Ruichen Shao",
        "Bei Li",
        "Gangao Liu",
        "Yang Chen",
        "Xiang Zhou",
        "Jingang Wang",
        "Xunliang Cai",
        "Peng Li"
      ],
      "summary": "Direct Preference Optimization (DPO) has gained attention as an efficient alternative to reinforcement learning from human feedback (RLHF) for aligning large language models (LLMs) with human preferences. Despite its advantages, DPO suffers from a length bias, generating responses longer than those from the reference model. Existing solutions like SimPO and SamPO address this issue but uniformly treat the contribution of rewards across sequences, overlooking temporal dynamics. To this end, we propose an enhanced preference optimization method that incorporates a temporal decay factor controlled by a gamma parameter. This dynamic weighting mechanism adjusts the influence of each reward based on its position in the sequence, prioritizing earlier tokens that are more critical for alignment. By adaptively focusing on more relevant feedback, our approach mitigates overfitting to less pertinent data and remains responsive to evolving human preferences. Experimental results on several benchmarks show that our approach consistently outperforms vanilla DPO by 5.9-8.8 points on AlpacaEval 2 and 3.3-9.7 points on Arena-Hard across different model architectures and sizes. Furthermore, additional experiments on mathematical and reasoning benchmarks (MMLU, GSM8K, and MATH) confirm that our method enhances performance without compromising general capabilities. Our codebase would be available at \\url{https://github.com/LotuSrc/D2PO}.",
      "arxiv_id": "2502.14340v1",
      "published": "2025-02-20",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2502.14340v1",
      "pdf_url": "https://arxiv.org/pdf/2502.14340v1.pdf"
    },
    {
      "title": "Improving LLM Safety and Helpfulness using SFT and DPO: A Study on OPT-350M",
      "authors": [
        "Piyush Pant"
      ],
      "summary": "This research investigates the effectiveness of alignment techniques, Supervised Fine-Tuning (SFT), Direct Preference Optimization (DPO), and a combined SFT+DPO approach on improving the safety and helpfulness of the OPT-350M language model. Utilizing the Anthropic Helpful-Harmless RLHF dataset, we train and evaluate four models: the base OPT350M, an SFT model, a DPO model, and a model trained with both SFT and DPO. We introduce three key evaluation metrics: Harmlessness Rate (HmR), Helpfulness Rate (HpR), and a Combined Alignment Score (CAS), all derived from reward model outputs. The results show that while SFT outperforms DPO, The combined SFT+DPO model outperforms all others across all metrics, demonstrating the complementary nature of these techniques. Our findings also highlight challenges posed by noisy data, limited GPU resources, and training constraints. This study offers a comprehensive view of how fine-tuning strategies affect model alignment and provides a foundation for more robust alignment pipelines in future work.",
      "arxiv_id": "2509.09055v1",
      "published": "2025-09-10",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2509.09055v1",
      "pdf_url": "https://arxiv.org/pdf/2509.09055v1.pdf"
    },
    {
      "title": "Cross-lingual Human-Preference Alignment for Neural Machine Translation with Direct Quality Optimization",
      "authors": [
        "Kaden Uhlig",
        "Joern Wuebker",
        "Raphael Reinauer",
        "John DeNero"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) and derivative techniques like Direct Preference Optimization (DPO) are task-alignment algorithms used to repurpose general, foundational models for specific tasks. We show that applying task-alignment to neural machine translation (NMT) addresses an existing task--data mismatch in NMT, leading to improvements across all languages of a multilingual model, even when task-alignment is only applied to a subset of those languages. We do so by introducing Direct Quality Optimization (DQO), a variant of DPO leveraging a pre-trained translation quality estimation model as a proxy for human preferences, and verify the improvements with both automatic metrics and human evaluation.",
      "arxiv_id": "2409.17673v3",
      "published": "2024-09-26",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2409.17673v3",
      "pdf_url": "https://arxiv.org/pdf/2409.17673v3.pdf"
    },
    {
      "title": "Off-Policy Corrected Reward Modeling for Reinforcement Learning from Human Feedback",
      "authors": [
        "Johannes Ackermann",
        "Takashi Ishida",
        "Masashi Sugiyama"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) allows us to train models, such as language models (LMs), to follow complex human preferences. In RLHF for LMs, we first train an LM using supervised fine-tuning, sample pairs of responses, obtain human feedback, and use the resulting data to train a reward model (RM). RL methods are then used to train the LM to maximize the reward given by the RM. As training progresses, the responses generated by the LM no longer resemble the responses seen by the RM during training, leading to the RM becoming inaccurate. The score given by the RM keeps increasing, but the learned behavior no longer matches the human preferences. This issue is known as overoptimization. We investigate overoptimization from the point of view of distribution shift and show that the shift results in an inconsistent estimate of the RM parameters, leading to an inconsistent estimate of the policy gradient. We propose Off-Policy Corrected Reward Modeling (OCRM), which iteratively off-policy corrects the RM using importance weighting, without requiring new labels or samples. This results in a more accurate RM, which empirically leads to an improved final policy. We validate our approach in experiments with summarization and chatbot datasets and show that it performs significantly better than standard RLHF methods and baselines. Our implementation is available at https://github.com/JohannesAck/OffPolicyCorrectedRewardModeling",
      "arxiv_id": "2507.15507v1",
      "published": "2025-07-21",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2507.15507v1",
      "pdf_url": "https://arxiv.org/pdf/2507.15507v1.pdf"
    },
    {
      "title": "ChatGLM-RLHF: Practices of Aligning Large Language Models with Human Feedback",
      "authors": [
        "Zhenyu Hou",
        "Yilin Niu",
        "Zhengxiao Du",
        "Xiaohan Zhang",
        "Xiao Liu",
        "Aohan Zeng",
        "Qinkai Zheng",
        "Minlie Huang",
        "Hongning Wang",
        "Jie Tang",
        "Yuxiao Dong"
      ],
      "summary": "ChatGLM is a free-to-use AI service powered by the ChatGLM family of large language models (LLMs). In this paper, we present the ChatGLM-RLHF pipeline -- a reinforcement learning from human feedback (RLHF) system -- designed to enhance ChatGLM's alignment with human preferences. ChatGLM-RLHF encompasses three major components: the collection of human preference data, the training of the reward model, and the optimization of policies. Throughout the process of integrating ChatGLM-RLHF into production, we encountered and addressed several unprecedented challenges. We introduce the strategies to mitigate reward variance for stabilized large-scale training, implement model parallelism with fused gradient-descent, and design regularization constraints to avoid catastrophic forgetting in LLMs. Experiments show that ChatGLM-RLHF brings significant improvements in alignment tasks compared to the supervised fine-tuned (SFT) version of ChatGLM. For instance, it achieves on average 15\\% more wins against ChatGLM-SFT in Chinese alignment tasks. The work presents our practices of aligning LLMs with human preferences, offering insights into the challenges and solutions in RLHF implementations.",
      "arxiv_id": "2404.00934v2",
      "published": "2024-04-01",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2404.00934v2",
      "pdf_url": "https://arxiv.org/pdf/2404.00934v2.pdf"
    },
    {
      "title": "Mapping out the Space of Human Feedback for Reinforcement Learning: A Conceptual Framework",
      "authors": [
        "Yannick Metz",
        "David Lindner",
        "Rapha\u00ebl Baur",
        "Mennatallah El-Assady"
      ],
      "summary": "Reinforcement Learning from Human feedback (RLHF) has become a powerful tool to fine-tune or train agentic machine learning models. Similar to how humans interact in social contexts, we can use many types of feedback to communicate our preferences, intentions, and knowledge to an RL agent. However, applications of human feedback in RL are often limited in scope and disregard human factors. In this work, we bridge the gap between machine learning and human-computer interaction efforts by developing a shared understanding of human feedback in interactive learning scenarios. We first introduce a taxonomy of feedback types for reward-based learning from human feedback based on nine key dimensions. Our taxonomy allows for unifying human-centered, interface-centered, and model-centered aspects. In addition, we identify seven quality metrics of human feedback influencing both the human ability to express feedback and the agent's ability to learn from the feedback. Based on the feedback taxonomy and quality criteria, we derive requirements and design choices for systems learning from human feedback. We relate these requirements and design choices to existing work in interactive machine learning. In the process, we identify gaps in existing work and future research opportunities. We call for interdisciplinary collaboration to harness the full potential of reinforcement learning with data-driven co-adaptive modeling and varied interaction mechanics.",
      "arxiv_id": "2411.11761v2",
      "published": "2024-11-18",
      "categories": [
        "cs.LG",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2411.11761v2",
      "pdf_url": "https://arxiv.org/pdf/2411.11761v2.pdf"
    },
    {
      "title": "Group Robust Preference Optimization in Reward-free RLHF",
      "authors": [
        "Shyam Sundhar Ramesh",
        "Yifan Hu",
        "Iason Chaimalas",
        "Viraj Mehta",
        "Pier Giuseppe Sessa",
        "Haitham Bou Ammar",
        "Ilija Bogunovic"
      ],
      "summary": "Adapting large language models (LLMs) for specific tasks usually involves fine-tuning through reinforcement learning with human feedback (RLHF) on preference data. While these data often come from diverse labelers' groups (e.g., different demographics, ethnicities, company teams, etc.), traditional RLHF approaches adopt a \"one-size-fits-all\" approach, i.e., they indiscriminately assume and optimize a single preference model, thus not being robust to unique characteristics and needs of the various groups. To address this limitation, we propose a novel Group Robust Preference Optimization (GRPO) method to align LLMs to individual groups' preferences robustly. Our approach builds upon reward-free direct preference optimization methods, but unlike previous approaches, it seeks a robust policy which maximizes the worst-case group performance. To achieve this, GRPO adaptively and sequentially weights the importance of different groups, prioritizing groups with worse cumulative loss. We theoretically study the feasibility of GRPO and analyze its convergence for the log-linear policy class. By fine-tuning LLMs with GRPO using diverse group-based global opinion data, we significantly improved performance for the worst-performing groups, reduced loss imbalances across groups, and improved probability accuracies compared to non-robust baselines.",
      "arxiv_id": "2405.20304v1",
      "published": "2024-05-30",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.20304v1",
      "pdf_url": "https://arxiv.org/pdf/2405.20304v1.pdf"
    },
    {
      "title": "Uncertainty-Penalized Reinforcement Learning from Human Feedback with Diverse Reward LoRA Ensembles",
      "authors": [
        "Yuanzhao Zhai",
        "Han Zhang",
        "Yu Lei",
        "Yue Yu",
        "Kele Xu",
        "Dawei Feng",
        "Bo Ding",
        "Huaimin Wang"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) emerges as a promising paradigm for aligning large language models (LLMs). However, a notable challenge in RLHF is overoptimization, where beyond a certain threshold, the pursuit of higher rewards leads to a decline in human preferences. In this paper, we observe the weakness of KL regularization which is commonly employed in existing RLHF methods to address overoptimization. To mitigate this limitation, we scrutinize the RLHF objective in the offline dataset and propose uncertainty-penalized RLHF (UP-RLHF), which incorporates uncertainty regularization during RL-finetuning. To enhance the uncertainty quantification abilities for reward models, we first propose a diverse low-rank adaptation (LoRA) ensemble by maximizing the nuclear norm of LoRA matrix concatenations. Then we optimize policy models utilizing penalized rewards, determined by both rewards and uncertainties provided by the diverse reward LoRA ensembles. Our experimental results, based on two real human preference datasets, showcase the effectiveness of diverse reward LoRA ensembles in quantifying reward uncertainty. Additionally, uncertainty regularization in UP-RLHF proves to be pivotal in mitigating overoptimization, thereby contributing to the overall performance.",
      "arxiv_id": "2401.00243v1",
      "published": "2023-12-30",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2401.00243v1",
      "pdf_url": "https://arxiv.org/pdf/2401.00243v1.pdf"
    },
    {
      "title": "Policy Teaching via Data Poisoning in Learning from Human Preferences",
      "authors": [
        "Andi Nika",
        "Jonathan N\u00f6ther",
        "Debmalya Mandal",
        "Parameswaran Kamalaruban",
        "Adish Singla",
        "Goran Radanovi\u0107"
      ],
      "summary": "We study data poisoning attacks in learning from human preferences. More specifically, we consider the problem of teaching/enforcing a target policy $\u03c0^\\dagger$ by synthesizing preference data. We seek to understand the susceptibility of different preference-based learning paradigms to poisoned preference data by analyzing the number of samples required by the attacker to enforce $\u03c0^\\dagger$. We first propose a general data poisoning formulation in learning from human preferences and then study it for two popular paradigms, namely: (a) reinforcement learning from human feedback (RLHF) that operates by learning a reward model using preferences; (b) direct preference optimization (DPO) that directly optimizes policy using preferences. We conduct a theoretical analysis of the effectiveness of data poisoning in a setting where the attacker is allowed to augment a pre-existing dataset and also study its special case where the attacker can synthesize the entire preference dataset from scratch. As our main results, we provide lower/upper bounds on the number of samples required to enforce $\u03c0^\\dagger$. Finally, we discuss the implications of our results in terms of the susceptibility of these learning paradigms under such data poisoning attacks.",
      "arxiv_id": "2503.10228v1",
      "published": "2025-03-13",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.10228v1",
      "pdf_url": "https://arxiv.org/pdf/2503.10228v1.pdf"
    },
    {
      "title": "AnnoDPO: Protein Functional Annotation Learning with Direct Preference Optimization",
      "authors": [
        "Zixuan Jiang",
        "Renjing Xu"
      ],
      "summary": "Deciphering protein function remains a fundamental challenge in protein representation learning. The task presents significant difficulties for protein language models (PLMs) due to the sheer volume of functional annotation categories and the highly imbalanced distribution of annotated instances across biological ontologies. Inspired by the remarkable success of reinforcement learning from human feedback (RLHF) in large language model (LLM) alignment, we propose AnnoDPO, a novel multi-modal framework for protein function prediction that leverages Direct Preference Optimization (DPO) to enhance annotation learning. Our methodology addresses the dual challenges of annotation scarcity and category imbalance through preference-aligned training objectives, establishing a new paradigm for biological knowledge integration in protein representation learning.",
      "arxiv_id": "2506.07035v1",
      "published": "2025-06-08",
      "categories": [
        "q-bio.BM",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.07035v1",
      "pdf_url": "https://arxiv.org/pdf/2506.07035v1.pdf"
    },
    {
      "title": "RLHF from Heterogeneous Feedback via Personalization and Preference Aggregation",
      "authors": [
        "Chanwoo Park",
        "Mingyang Liu",
        "Dingwen Kong",
        "Kaiqing Zhang",
        "Asuman Ozdaglar"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has been an effective technique for aligning AI systems with human values, with remarkable successes in fine-tuning large-language models recently. Most existing RLHF paradigms make the underlying assumption that human preferences are relatively homogeneous, and can be encoded by a single reward model. In this paper, we focus on addressing the issues due to the inherent heterogeneity in human preferences, as well as their potential strategic behavior in providing feedback. Specifically, we propose two frameworks to address heterogeneous human feedback in principled ways: personalization-based one and aggregation-based one. For the former, we propose two approaches based on representation learning and clustering, respectively, for learning multiple reward models that trades off the bias (due to preference heterogeneity) and variance (due to the use of fewer data for learning each model by personalization). We then establish sample complexity guarantees for both approaches. For the latter, we aim to adhere to the single-model framework, as already deployed in the current RLHF paradigm, by carefully aggregating diverse and truthful preferences from humans. We propose two approaches based on reward and preference aggregation, respectively: the former utilizes both utilitarianism and Leximin approaches to aggregate individual reward models, with sample complexity guarantees; the latter directly aggregates the human feedback in the form of probabilistic opinions. Under the probabilistic-opinion-feedback model, we also develop an approach to handle strategic human labelers who may bias and manipulate the aggregated preferences with untruthful feedback. Based on the ideas in mechanism design, our approach ensures truthful preference reporting, with the induced aggregation rule maximizing social welfare functions.",
      "arxiv_id": "2405.00254v2",
      "published": "2024-04-30",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.00254v2",
      "pdf_url": "https://arxiv.org/pdf/2405.00254v2.pdf"
    },
    {
      "title": "Generalizing Reward Modeling for Out-of-Distribution Preference Learning",
      "authors": [
        "Chen Jia"
      ],
      "summary": "Preference learning (PL) with large language models (LLMs) aims to align the LLMs' generations with human preferences. Previous work on reinforcement learning from human feedback (RLHF) has demonstrated promising results in in-distribution PL. However, due to the difficulty of obtaining human feedback, discretely training reward models for every encountered distribution is challenging. Thus, out-of-distribution (OOD) PL is practically useful for enhancing the generalization ability of LLMs with limited preference feedback. This work addresses OOD PL by optimizing a general reward model through a meta-learning approach. During meta-training, a bilevel optimization algorithm is utilized to learn a reward model capable of guiding policy learning to align with human preferences across various distributions. When encountering a test distribution, the meta-test procedure conducts regularized policy optimization using the learned reward model for PL. We theoretically demonstrate the convergence rate of the bilevel optimization algorithm under reasonable assumptions. Additionally, we conduct experiments on two text generation tasks across 20 held-out domains and outperform a variety of strong baselines across various evaluation metrics.",
      "arxiv_id": "2402.14760v2",
      "published": "2024-02-22",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2402.14760v2",
      "pdf_url": "https://arxiv.org/pdf/2402.14760v2.pdf"
    },
    {
      "title": "Direct Preference Optimization With Unobserved Preference Heterogeneity: The Necessity of Ternary Preferences",
      "authors": [
        "Keertana Chidambaram",
        "Karthik Vinay Seetharaman",
        "Vasilis Syrgkanis"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has become central to aligning large language models with human values, typically by first learning a reward model from preference data which is then used to update the model with reinforcement learning. Recent alternatives such as Direct Preference Optimization (DPO) simplify this pipeline by directly optimizing on preferences. However, both approaches often assume uniform annotator preferences and rely on binary comparisons, overlooking two key limitations: the diversity of human evaluators and the limitations of pairwise feedback. In this work, we address both these issues. First, we connect preference learning in RLHF with the econometrics literature and show that binary comparisons are insufficient for identifying latent user preferences from finite user data and infinite users, while (even incomplete) rankings over three or more responses ensure identifiability. Second, we introduce methods to incorporate heterogeneous preferences into alignment algorithms. We develop an Expectation-Maximization adaptation of DPO that discovers latent annotator types and trains a mixture of LLMs accordingly. Then we propose an aggregation algorithm using a min-max regret fairness criterion to produce a single generative policy with equitable performance guarantees. Together, these contributions establish a theoretical and algorithmic framework for fairness and personalization for diverse users in generative model alignment.",
      "arxiv_id": "2405.15065v2",
      "published": "2024-05-23",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.15065v2",
      "pdf_url": "https://arxiv.org/pdf/2405.15065v2.pdf"
    },
    {
      "title": "Filtered Direct Preference Optimization",
      "authors": [
        "Tetsuro Morimura",
        "Mitsuki Sakamoto",
        "Yuu Jinnai",
        "Kenshi Abe",
        "Kaito Ariu"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) plays a crucial role in aligning language models with human preferences. While the significance of dataset quality is generally recognized, explicit investigations into its impact within the RLHF framework, to our knowledge, have been limited. This paper addresses the issue of text quality within the preference dataset by focusing on direct preference optimization (DPO), an increasingly adopted reward-model-free RLHF method. We confirm that text quality significantly influences the performance of models optimized with DPO more than those optimized with reward-model-based RLHF. Building on this new insight, we propose an extension of DPO, termed filtered direct preference optimization (fDPO). fDPO uses a trained reward model to monitor the quality of texts within the preference dataset during DPO training. Samples of lower quality are discarded based on comparisons with texts generated by the model being optimized, resulting in a more accurate dataset. Experimental results demonstrate that fDPO enhances the final model performance. Our code is available at https://github.com/CyberAgentAILab/filtered-dpo.",
      "arxiv_id": "2404.13846v4",
      "published": "2024-04-22",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2404.13846v4",
      "pdf_url": "https://arxiv.org/pdf/2404.13846v4.pdf"
    },
    {
      "title": "A Minimaximalist Approach to Reinforcement Learning from Human Feedback",
      "authors": [
        "Gokul Swamy",
        "Christoph Dann",
        "Rahul Kidambi",
        "Zhiwei Steven Wu",
        "Alekh Agarwal"
      ],
      "summary": "We present Self-Play Preference Optimization (SPO), an algorithm for reinforcement learning from human feedback. Our approach is minimalist in that it does not require training a reward model nor unstable adversarial training and is therefore rather simple to implement. Our approach is maximalist in that it provably handles non-Markovian, intransitive, and stochastic preferences while being robust to the compounding errors that plague offline approaches to sequential prediction. To achieve the preceding qualities, we build upon the concept of a Minimax Winner (MW), a notion of preference aggregation from the social choice theory literature that frames learning from preferences as a zero-sum game between two policies. By leveraging the symmetry of this game, we prove that rather than using the traditional technique of dueling two policies to compute the MW, we can simply have a single agent play against itself while maintaining strong convergence guarantees. Practically, this corresponds to sampling multiple trajectories from a policy, asking a preference or teacher model to compare them, and then using the proportion of wins as the reward for a particular trajectory. We demonstrate that on a suite of continuous control tasks, we are able to learn significantly more efficiently than reward-model based approaches while maintaining robustness to the intransitive and stochastic preferences that frequently occur in practice when aggregating human judgments.",
      "arxiv_id": "2401.04056v2",
      "published": "2024-01-08",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2401.04056v2",
      "pdf_url": "https://arxiv.org/pdf/2401.04056v2.pdf"
    },
    {
      "title": "Adaptive Margin RLHF via Preference over Preferences",
      "authors": [
        "Yaswanth Chittepu",
        "Prasann Singhal",
        "Greg Durrett",
        "Scott Niekum"
      ],
      "summary": "Margin-based optimization is fundamental to improving generalization and robustness in classification tasks. In the context of reward model learning from preferences within Reinforcement Learning from Human Feedback (RLHF), existing methods typically rely on no margins, fixed margins, or margins that are simplistic functions of preference ratings. However, such formulations often fail to account for the varying strengths of different preferences, for example some preferences are associated with larger margins between responses, or they rely on noisy margin information derived from ratings. We argue that modeling the strength of preferences can lead to better generalization and more faithful alignment. Furthermore, many existing methods that use adaptive margins assume access to accurate preference scores, which can be difficult for humans to provide reliably. We propose an approach that leverages preferences over preferences, that is annotations indicating which of two preferences reflects a stronger distinction. We use this ordinal signal to infer adaptive margins on a per-datapoint basis. We introduce an extension to Direct Preference Optimization (DPO), DPO-PoP, that incorporates adaptive margins from preference-over-preference supervision, enabling improved discriminative and generative performance. Empirically, our method outperforms vanilla DPO, DPO with fixed margins, and DPO with ground-truth margins on the UltraFeedback dataset. Additionally, we show that there is a tradeoff between discriminative and generative performance: improving test classification accuracy, particularly by correctly labeling weaker preferences at the expense of stronger ones, can lead to a decline in generative quality. To navigate this tradeoff, we propose two sampling strategies to gather preference-over-preference labels: one favoring discriminative performance and one favoring generative performance.",
      "arxiv_id": "2509.22851v2",
      "published": "2025-09-26",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.22851v2",
      "pdf_url": "https://arxiv.org/pdf/2509.22851v2.pdf"
    },
    {
      "title": "Preference as Reward, Maximum Preference Optimization with Importance Sampling",
      "authors": [
        "Zaifan Jiang",
        "Xing Huang",
        "Chao Wei"
      ],
      "summary": "Preference learning is a key technology for aligning language models with human values. Reinforcement Learning from Human Feedback (RLHF) is a model-based algorithm to optimize preference learning, which first fits a reward model for preference scores and then optimizes the generating policy with an on-policy PPO algorithm to maximize the reward. The processing of RLHF is complex, time-consuming, and unstable. The Direct Preference Optimization (DPO) algorithm uses an off-policy algorithm to directly optimize the generating policy and eliminates the need for a reward model. DPO is more data-efficient and stable. However, DPO has a drawback of overfitting to the preference data and ignoring the KL-regularization term when the preference is deterministic. Identity mapping Preference Optimization(IPO) uses a root-finding MSE loss to incorporate KL-regularization. However, both DPO and IPO fail to properly address the KL-regularization term because the support of the preference distribution is not equal to the reference distribution. In this paper, we propose a simple and intuitive off-policy preference optimization algorithm from an importance sampling view, which we call Maximum Preference Optimization (MPO). MPO incorporates the off-policy KL-regularization term, making regularization truly effective. MPO achieves the best of both worlds by combining the objectives of RLHF and IPO while being an off-policy algorithm. Furthermore, MPO eliminates the need for a reward model and reference policy, simplifying the learning process and reducing memory usage.",
      "arxiv_id": "2312.16430v5",
      "published": "2023-12-27",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2312.16430v5",
      "pdf_url": "https://arxiv.org/pdf/2312.16430v5.pdf"
    },
    {
      "title": "Reveal the Mystery of DPO: The Connection between DPO and RL Algorithms",
      "authors": [
        "Xuerui Su",
        "Yue Wang",
        "Jinhua Zhu",
        "Mingyang Yi",
        "Feng Xu",
        "Zhiming Ma",
        "Yuting Liu"
      ],
      "summary": "With the rapid development of Large Language Models (LLMs), numerous Reinforcement Learning from Human Feedback (RLHF) algorithms have been introduced to improve model safety and alignment with human preferences. These algorithms can be divided into two main frameworks based on whether they require an explicit reward (or value) function for training: actor-critic-based Proximal Policy Optimization (PPO) and alignment-based Direct Preference Optimization (DPO). The mismatch between DPO and PPO, such as DPO's use of a classification loss driven by human-preferred data, has raised confusion about whether DPO should be classified as a Reinforcement Learning (RL) algorithm. To address these ambiguities, we focus on three key aspects related to DPO, RL, and other RLHF algorithms: (1) the construction of the loss function; (2) the target distribution at which the algorithm converges; (3) the impact of key components within the loss function. Specifically, we first establish a unified framework named UDRRA connecting these algorithms based on the construction of their loss functions. Next, we uncover their target policy distributions within this framework. Finally, we investigate the critical components of DPO to understand their impact on the convergence rate. Our work provides a deeper understanding of the relationship between DPO, RL, and other RLHF algorithms, offering new insights for improving existing algorithms.",
      "arxiv_id": "2502.03095v1",
      "published": "2025-02-05",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2502.03095v1",
      "pdf_url": "https://arxiv.org/pdf/2502.03095v1.pdf"
    },
    {
      "title": "Accelerated Preference Optimization for Large Language Model Alignment",
      "authors": [
        "Jiafan He",
        "Huizhuo Yuan",
        "Quanquan Gu"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a pivotal tool for aligning large language models (LLMs) with human preferences. Direct Preference Optimization (DPO), one of the most popular approaches, formulates RLHF as a policy optimization problem without explicitly estimating the reward function. It overcomes the stability and efficiency issues of two-step approaches, which typically involve first estimating the reward function and then optimizing the policy via proximal policy optimization (PPO). Since RLHF is essentially an optimization problem, and it is well-known that momentum techniques can accelerate optimization both theoretically and empirically, a natural question arises: Can RLHF be accelerated by momentum? This paper answers this question in the affirmative. In detail, we first show that the iterative preference optimization method can be viewed as a proximal point method. Based on this observation, we propose a general Accelerated Preference Optimization (APO) framework, which unifies many existing preference optimization algorithms and employs Nesterov's momentum technique to speed up the alignment of LLMs. Theoretically, we demonstrate that APO can achieve a faster convergence rate than the standard iterative preference optimization methods, including DPO and Self-Play Preference Optimization (SPPO). Empirically, we show the superiority of APO over DPO, iterative DPO, and other strong baselines for RLHF on the AlpacaEval 2.0 benchmark.",
      "arxiv_id": "2410.06293v1",
      "published": "2024-10-08",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2410.06293v1",
      "pdf_url": "https://arxiv.org/pdf/2410.06293v1.pdf"
    },
    {
      "title": "RRHF: Rank Responses to Align Language Models with Human Feedback without tears",
      "authors": [
        "Zheng Yuan",
        "Hongyi Yuan",
        "Chuanqi Tan",
        "Wei Wang",
        "Songfang Huang",
        "Fei Huang"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) facilitates the alignment of large language models with human preferences, significantly enhancing the quality of interactions between humans and models. InstructGPT implements RLHF through several stages, including Supervised Fine-Tuning (SFT), reward model training, and Proximal Policy Optimization (PPO). However, PPO is sensitive to hyperparameters and requires multiple models in its standard implementation, making it hard to train and scale up to larger parameter counts. In contrast, we propose a novel learning paradigm called RRHF, which scores sampled responses from different sources via a logarithm of conditional probabilities and learns to align these probabilities with human preferences through ranking loss. RRHF can leverage sampled responses from various sources including the model responses from itself, other large language model responses, and human expert responses to learn to rank them. RRHF only needs 1 to 2 models during tuning and can efficiently align language models with human preferences robustly without complex hyperparameter tuning. Additionally, RRHF can be considered an extension of SFT and reward model training while being simpler than PPO in terms of coding, model counts, and hyperparameters. We evaluate RRHF on the Helpful and Harmless dataset, demonstrating comparable alignment performance with PPO by reward model score and human labeling. Extensive experiments show that the performance of RRHF is highly related to sampling quality which suggests RRHF is a best-of-n learner. Codes available at https://github.com/GanjinZero/RRHF.",
      "arxiv_id": "2304.05302v3",
      "published": "2023-04-11",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2304.05302v3",
      "pdf_url": "https://arxiv.org/pdf/2304.05302v3.pdf"
    },
    {
      "title": "Sample-Efficient Reinforcement Learning from Human Feedback via Information-Directed Sampling",
      "authors": [
        "Han Qi",
        "Haochen Yang",
        "Qiaosheng Zhang",
        "Zhuoran Yang"
      ],
      "summary": "We study the problem of reinforcement learning from human feedback (RLHF), a critical problem in training large language models, from a theoretical perspective. Our main contribution is the design of novel sample-efficient RLHF algorithms based on information-directed sampling (IDS), an online decision-making principle inspired by information theory. Our algorithms maximize the sum of the value function and a mutual information term that encourages exploration of the unknown environment (which quantifies the information gained about the environment through observed human feedback data). To tackle the challenge of large state spaces and improve sample efficiency, we construct a simplified \\emph{surrogate environment} and introduce a novel distance measure (named the \\emph{$\\ell_g$-distance}), enabling our IDS-based algorithm to achieve a Bayesian regret upper bound of order $O(H^{\\frac{3}{2}}\\sqrt{\\log(K(\u03b5)) T})$, where $H$ is the episode length, $T$ is the number of episode and $K(\u03b5)$ is related to the covering number of the environment. Specializing to the tabular settings, this regret bound is of order $\\tilde{O}(H^2\\sqrt{SAT})$, where $S$ and $A$ are the numbers of states and actions. Finally, we propose an Approximate-IDS algorithm that is computationally more efficient while maintaining nearly the same sample efficiency. The design principle of this approximate algorithm is not only effective in RLHF settings but also applicable to the standard RL framework. Moreover, our work showcases the value of information theory in reinforcement learning and in the training of large language models.",
      "arxiv_id": "2502.05434v3",
      "published": "2025-02-08",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2502.05434v3",
      "pdf_url": "https://arxiv.org/pdf/2502.05434v3.pdf"
    },
    {
      "title": "Aligning Humans and Robots via Reinforcement Learning from Implicit Human Feedback",
      "authors": [
        "Suzie Kim",
        "Hye-Bin Shin",
        "Seong-Whan Lee"
      ],
      "summary": "Conventional reinforcement learning (RL) ap proaches often struggle to learn effective policies under sparse reward conditions, necessitating the manual design of complex, task-specific reward functions. To address this limitation, rein forcement learning from human feedback (RLHF) has emerged as a promising strategy that complements hand-crafted rewards with human-derived evaluation signals. However, most existing RLHF methods depend on explicit feedback mechanisms such as button presses or preference labels, which disrupt the natural interaction process and impose a substantial cognitive load on the user. We propose a novel reinforcement learning from implicit human feedback (RLIHF) framework that utilizes non-invasive electroencephalography (EEG) signals, specifically error-related potentials (ErrPs), to provide continuous, implicit feedback without requiring explicit user intervention. The proposed method adopts a pre-trained decoder to transform raw EEG signals into probabilistic reward components, en abling effective policy learning even in the presence of sparse external rewards. We evaluate our approach in a simulation environment built on the MuJoCo physics engine, using a Kinova Gen2 robotic arm to perform a complex pick-and-place task that requires avoiding obstacles while manipulating target objects. The results show that agents trained with decoded EEG feedback achieve performance comparable to those trained with dense, manually designed rewards. These findings validate the potential of using implicit neural feedback for scalable and human-aligned reinforcement learning in interactive robotics.",
      "arxiv_id": "2507.13171v1",
      "published": "2025-07-17",
      "categories": [
        "cs.RO",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2507.13171v1",
      "pdf_url": "https://arxiv.org/pdf/2507.13171v1.pdf"
    },
    {
      "title": "Governance Challenges in Reinforcement Learning from Human Feedback: Evaluator Rationality and Reinforcement Stability",
      "authors": [
        "Dana Alsagheer",
        "Abdulrahman Kamal",
        "Mohammad Kamal",
        "Weidong Shi"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is central in aligning large language models (LLMs) with human values and expectations. However, the process remains susceptible to governance challenges, including evaluator bias, inconsistency, and the unreliability of feedback. This study examines how the cognitive capacity of evaluators, specifically their level of rationality, affects the stability of reinforcement signals. A controlled experiment comparing high-rationality and low-rationality participants reveals that evaluators with higher rationality scores produce significantly more consistent and expert-aligned feedback. In contrast, lower-rationality participants demonstrate considerable variability in their reinforcement decisions ($p < 0.01$). To address these challenges and improve RLHF governance, we recommend implementing evaluator pre-screening, systematic auditing of feedback consistency, and reliability-weighted reinforcement aggregation. These measures enhance the fairness, transparency, and robustness of AI alignment pipelines.",
      "arxiv_id": "2504.13972v1",
      "published": "2025-04-17",
      "categories": [
        "cs.CY",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2504.13972v1",
      "pdf_url": "https://arxiv.org/pdf/2504.13972v1.pdf"
    },
    {
      "title": "Active teacher selection for reinforcement learning from human feedback",
      "authors": [
        "Rachel Freedman",
        "Justin Svegliato",
        "Kyle Wray",
        "Stuart Russell"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) enables machine learning systems to learn objectives from human feedback. A core limitation of these systems is their assumption that all feedback comes from a single human teacher, despite querying a range of distinct teachers. We propose the Hidden Utility Bandit (HUB) framework to model differences in teacher rationality, expertise, and costliness, formalizing the problem of learning from multiple teachers. We develop a variety of solution algorithms and apply them to two real-world domains: paper recommendation systems and COVID-19 vaccine testing. We find that the Active Teacher Selection (ATS) algorithm outperforms baseline algorithms by actively selecting when and which teacher to query. The HUB framework and ATS algorithm demonstrate the importance of leveraging differences between teachers to learn accurate reward models, facilitating future research on active teacher selection for robust reward modeling.",
      "arxiv_id": "2310.15288v2",
      "published": "2023-10-23",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2310.15288v2",
      "pdf_url": "https://arxiv.org/pdf/2310.15288v2.pdf"
    },
    {
      "title": "Contextual Online Uncertainty-Aware Preference Learning for Human Feedback",
      "authors": [
        "Nan Lu",
        "Ethan X. Fang",
        "Junwei Lu"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has become a pivotal paradigm in artificial intelligence to align large models with human preferences. In this paper, we propose a novel statistical framework to simultaneously conduct the online decision-making and statistical inference on the optimal model using human preference data based on dynamic contextual information. Our approach introduces an efficient decision strategy that achieves both the optimal regret bound and the asymptotic distribution of the estimators. A key challenge in RLHF is handling the dependent online human preference outcomes with dynamic contexts. To address this, in the methodological aspect, we propose a two-stage algorithm starting with $\u03b5$-greedy followed by exploitations; in the theoretical aspect, we tailor anti-concentration inequalities and matrix martingale concentration techniques to derive the uniform estimation rate and asymptotic normality of the estimators using dependent samples from both stages. Extensive simulation results demonstrate that our method outperforms state-of-the-art strategies. We apply the proposed framework to analyze the human preference data for ranking large language models on the Massive Multitask Language Understanding dataset, yielding insightful results on the performance of different large language models for medical anatomy knowledge.",
      "arxiv_id": "2504.19342v2",
      "published": "2025-04-27",
      "categories": [
        "stat.ML",
        "cs.LG",
        "stat.ME"
      ],
      "url": "https://arxiv.org/abs/2504.19342v2",
      "pdf_url": "https://arxiv.org/pdf/2504.19342v2.pdf"
    },
    {
      "title": "Aligning Visual Contrastive learning models via Preference Optimization",
      "authors": [
        "Amirabbas Afzali",
        "Borna Khodabandeh",
        "Ali Rasekh",
        "Mahyar JafariNodeh",
        "Sepehr kazemi",
        "Simon Gottschalk"
      ],
      "summary": "Contrastive learning models have demonstrated impressive abilities to capture semantic similarities by aligning representations in the embedding space. However, their performance can be limited by the quality of the training data and its inherent biases. While Preference Optimization (PO) methods such as Reinforcement Learning from Human Feedback (RLHF) and Direct Preference Optimization (DPO) have been applied to align generative models with human preferences, their use in contrastive learning has yet to be explored. This paper introduces a novel method for training contrastive learning models using different PO methods to break down complex concepts. Our method systematically aligns model behavior with desired preferences, enhancing performance on the targeted task. In particular, we focus on enhancing model robustness against typographic attacks and inductive biases, commonly seen in contrastive vision-language models like CLIP. Our experiments demonstrate that models trained using PO outperform standard contrastive learning techniques while retaining their ability to handle adversarial challenges and maintain accuracy on other downstream tasks. This makes our method well-suited for tasks requiring fairness, robustness, and alignment with specific preferences. We evaluate our method for tackling typographic attacks on images and explore its ability to disentangle gender concepts and mitigate gender bias, showcasing the versatility of our approach.",
      "arxiv_id": "2411.08923v3",
      "published": "2024-11-12",
      "categories": [
        "cs.CV",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2411.08923v3",
      "pdf_url": "https://arxiv.org/pdf/2411.08923v3.pdf"
    },
    {
      "title": "Iterative Nash Policy Optimization: Aligning LLMs with General Preferences via No-Regret Learning",
      "authors": [
        "Yuheng Zhang",
        "Dian Yu",
        "Baolin Peng",
        "Linfeng Song",
        "Ye Tian",
        "Mingyue Huo",
        "Nan Jiang",
        "Haitao Mi",
        "Dong Yu"
      ],
      "summary": "Reinforcement Learning with Human Feedback (RLHF) has achieved great success in aligning large language models (LLMs) with human preferences. Prevalent RLHF approaches are reward-based, following the Bradley-Terry (BT) model assumption, which may not fully capture the complexity of human preferences. In this paper, we explore RLHF under a general preference framework and approach it from a game-theoretic perspective. Specifically, we formulate the problem as a two-player game and propose a novel online algorithm, iterative Nash policy optimization (INPO). The key idea is to let the policy play against itself via no-regret learning, thereby approximating the Nash policy. Unlike previous methods, INPO bypasses the need for estimating the expected win rate for individual responses, which typically incurs high computational or annotation costs. Instead, we introduce a new loss objective that is directly minimized over a preference dataset. We provide theoretical analysis for our approach and demonstrate its effectiveness through experiments on various representative benchmarks. With an LLaMA-3-8B-based SFT model, INPO achieves a 42.6% length-controlled win rate on AlpacaEval 2.0 and a 37.8% win rate on Arena-Hard, showing substantial improvement over the state-of-the-art online RLHF algorithms.",
      "arxiv_id": "2407.00617v4",
      "published": "2024-06-30",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.GT"
      ],
      "url": "https://arxiv.org/abs/2407.00617v4",
      "pdf_url": "https://arxiv.org/pdf/2407.00617v4.pdf"
    },
    {
      "title": "Exploring Data Scaling Trends and Effects in Reinforcement Learning from Human Feedback",
      "authors": [
        "Wei Shen",
        "Guanlin Liu",
        "Zheng Wu",
        "Ruofei Zhu",
        "Qingping Yang",
        "Chao Xin",
        "Yu Yue",
        "Lin Yan"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is crucial for aligning large language models with human preferences. While recent research has focused on algorithmic improvements, the importance of prompt-data construction has been overlooked. This paper addresses this gap by exploring data-driven bottlenecks in RLHF performance scaling, particularly reward hacking and decreasing response diversity. We introduce a hybrid reward system combining reasoning task verifiers (RTV) and a generative reward model (GenRM) to mitigate reward hacking. We also propose a novel prompt-selection method, Pre-PPO, to maintain response diversity and enhance learning effectiveness. Additionally, we find that prioritizing mathematical and coding tasks early in RLHF training significantly improves performance. Experiments across two model sizes validate our methods' effectiveness and scalability. Results show that RTV is most resistant to reward hacking, followed by GenRM with ground truth, and then GenRM with SFT Best-of-N responses. Our strategies enable rapid capture of subtle task-specific distinctions, leading to substantial improvements in overall RLHF performance. This work highlights the importance of careful data construction and provides practical methods to overcome performance barriers in RLHF.",
      "arxiv_id": "2503.22230v3",
      "published": "2025-03-28",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.22230v3",
      "pdf_url": "https://arxiv.org/pdf/2503.22230v3.pdf"
    },
    {
      "title": "Uncertainty-Penalized Direct Preference Optimization",
      "authors": [
        "Sam Houliston",
        "Aliz\u00e9e Pace",
        "Alexander Immer",
        "Gunnar R\u00e4tsch"
      ],
      "summary": "Aligning Large Language Models (LLMs) to human preferences in content, style, and presentation is challenging, in part because preferences are varied, context-dependent, and sometimes inherently ambiguous. While successful, Reinforcement Learning from Human Feedback (RLHF) and Direct Preference Optimization (DPO) are prone to the issue of proxy reward overoptimization. Analysis of the DPO loss reveals a critical need for regularization for mislabeled or ambiguous preference pairs to avoid reward hacking. In this work, we develop a pessimistic framework for DPO by introducing preference uncertainty penalization schemes, inspired by offline reinforcement learning. The penalization serves as a correction to the loss which attenuates the loss gradient for uncertain samples. Evaluation of the methods is performed with GPT2 Medium on the Anthropic-HH dataset using a model ensemble to obtain uncertainty estimates, and shows improved overall performance compared to vanilla DPO, as well as better completions on prompts from high-uncertainty chosen/rejected responses.",
      "arxiv_id": "2410.20187v1",
      "published": "2024-10-26",
      "categories": [
        "cs.LG",
        "cs.AI",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2410.20187v1",
      "pdf_url": "https://arxiv.org/pdf/2410.20187v1.pdf"
    },
    {
      "title": "Open Problems and Fundamental Limitations of Reinforcement Learning from Human Feedback",
      "authors": [
        "Stephen Casper",
        "Xander Davies",
        "Claudia Shi",
        "Thomas Krendl Gilbert",
        "J\u00e9r\u00e9my Scheurer",
        "Javier Rando",
        "Rachel Freedman",
        "Tomasz Korbak",
        "David Lindner",
        "Pedro Freire",
        "Tony Wang",
        "Samuel Marks",
        "Charbel-Rapha\u00ebl Segerie",
        "Micah Carroll",
        "Andi Peng",
        "Phillip Christoffersen",
        "Mehul Damani",
        "Stewart Slocum",
        "Usman Anwar",
        "Anand Siththaranjan",
        "Max Nadeau",
        "Eric J. Michaud",
        "Jacob Pfau",
        "Dmitrii Krasheninnikov",
        "Xin Chen",
        "Lauro Langosco",
        "Peter Hase",
        "Erdem B\u0131y\u0131k",
        "Anca Dragan",
        "David Krueger",
        "Dorsa Sadigh",
        "Dylan Hadfield-Menell"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is a technique for training AI systems to align with human goals. RLHF has emerged as the central method used to finetune state-of-the-art large language models (LLMs). Despite this popularity, there has been relatively little public work systematizing its flaws. In this paper, we (1) survey open problems and fundamental limitations of RLHF and related methods; (2) overview techniques to understand, improve, and complement RLHF in practice; and (3) propose auditing and disclosure standards to improve societal oversight of RLHF systems. Our work emphasizes the limitations of RLHF and highlights the importance of a multi-faceted approach to the development of safer AI systems.",
      "arxiv_id": "2307.15217v2",
      "published": "2023-07-27",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2307.15217v2",
      "pdf_url": "https://arxiv.org/pdf/2307.15217v2.pdf"
    },
    {
      "title": "Learning Optimal Advantage from Preferences and Mistaking it for Reward",
      "authors": [
        "W. Bradley Knox",
        "Stephane Hatgis-Kessell",
        "Sigurdur Orn Adalgeirsson",
        "Serena Booth",
        "Anca Dragan",
        "Peter Stone",
        "Scott Niekum"
      ],
      "summary": "We consider algorithms for learning reward functions from human preferences over pairs of trajectory segments, as used in reinforcement learning from human feedback (RLHF). Most recent work assumes that human preferences are generated based only upon the reward accrued within those segments, or their partial return. Recent work casts doubt on the validity of this assumption, proposing an alternative preference model based upon regret. We investigate the consequences of assuming preferences are based upon partial return when they actually arise from regret. We argue that the learned function is an approximation of the optimal advantage function, $\\hat{A^*_r}$, not a reward function. We find that if a specific pitfall is addressed, this incorrect assumption is not particularly harmful, resulting in a highly shaped reward function. Nonetheless, this incorrect usage of $\\hat{A^*_r}$ is less desirable than the appropriate and simpler approach of greedy maximization of $\\hat{A^*_r}$. From the perspective of the regret preference model, we also provide a clearer interpretation of fine tuning contemporary large language models with RLHF. This paper overall provides insight regarding why learning under the partial return preference model tends to work so well in practice, despite it conforming poorly to how humans give preferences.",
      "arxiv_id": "2310.02456v1",
      "published": "2023-10-03",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2310.02456v1",
      "pdf_url": "https://arxiv.org/pdf/2310.02456v1.pdf"
    },
    {
      "title": "Preference Ranking Optimization for Human Alignment",
      "authors": [
        "Feifan Song",
        "Bowen Yu",
        "Minghao Li",
        "Haiyang Yu",
        "Fei Huang",
        "Yongbin Li",
        "Houfeng Wang"
      ],
      "summary": "Large language models (LLMs) often contain misleading content, emphasizing the need to align them with human values to ensure secure AI systems. Reinforcement learning from human feedback (RLHF) has been employed to achieve this alignment. However, it encompasses two main drawbacks: (1) RLHF exhibits complexity, instability, and sensitivity to hyperparameters in contrast to SFT. (2) Despite massive trial-and-error, multiple sampling is reduced to pair-wise contrast, thus lacking contrasts from a macro perspective. In this paper, we propose Preference Ranking Optimization (PRO) as an efficient SFT algorithm to directly fine-tune LLMs for human alignment. PRO extends the pair-wise contrast to accommodate preference rankings of any length. By iteratively contrasting candidates, PRO instructs the LLM to prioritize the best response while progressively ranking the rest responses. In this manner, PRO effectively transforms human alignment into aligning the probability ranking of n responses generated by LLM with the preference ranking of humans towards these responses. Experiments have shown that PRO outperforms baseline algorithms, achieving comparable results to ChatGPT and human responses through automatic-based, reward-based, GPT-4, and human evaluations.",
      "arxiv_id": "2306.17492v2",
      "published": "2023-06-30",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2306.17492v2",
      "pdf_url": "https://arxiv.org/pdf/2306.17492v2.pdf"
    },
    {
      "title": "Improving Reinforcement Learning from Human Feedback Using Contrastive Rewards",
      "authors": [
        "Wei Shen",
        "Xiaoying Zhang",
        "Yuanshun Yao",
        "Rui Zheng",
        "Hongyi Guo",
        "Yang Liu"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) is the mainstream paradigm used to align large language models (LLMs) with human preferences. Yet existing RLHF heavily relies on accurate and informative reward models, which are vulnerable and sensitive to noise from various sources, e.g. human labeling errors, making the pipeline fragile. In this work, we improve the effectiveness of the reward model by introducing a penalty term on the reward, named as \\textit{contrastive rewards}. %Contrastive rewards Our approach involves two steps: (1) an offline sampling step to obtain responses to prompts that serve as baseline calculation and (2) a contrastive reward calculated using the baseline responses and used in the Proximal Policy Optimization (PPO) step. We show that contrastive rewards enable the LLM to penalize reward uncertainty, improve robustness, encourage improvement over baselines, calibrate according to task difficulty, and reduce variance in PPO. We show empirically contrastive rewards can improve RLHF substantially, evaluated by both GPTs and humans, and our method consistently outperforms strong baselines.",
      "arxiv_id": "2403.07708v2",
      "published": "2024-03-12",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2403.07708v2",
      "pdf_url": "https://arxiv.org/pdf/2403.07708v2.pdf"
    },
    {
      "title": "UNA: Unifying Alignments of RLHF/PPO, DPO and KTO by a Generalized Implicit Reward Function",
      "authors": [
        "Zhichao Wang",
        "Bin Bi",
        "Can Huang",
        "Shiva Kumar Pentyala",
        "Zixu James Zhu",
        "Sitaram Asur",
        "Na Claire Cheng"
      ],
      "summary": "An LLM is pretrained on trillions of tokens, but the pretrained LLM may still generate undesired responses. To solve this problem, alignment techniques such as RLHF, DPO and KTO are proposed. However, these alignment techniques have limitations. For example, RLHF requires training the reward model and policy separately, which is complex, time-consuming, memory intensive and unstable during training processes. DPO proposes a mapping between an optimal policy and a reward, greatly simplifying the training process of RLHF. However, it can not take full advantages of a reward model and it is limited to pairwise preference data.   In this paper, we propose \\textbf{UN}ified \\textbf{A}lignment (UNA) which unifies RLHF/PPO, DPO and KTO. Firstly, we mathematically prove that given the classical RLHF objective, the optimal policy is induced by a generalize implicit reward function. With this novel mapping between a reward model and an optimal policy, UNA can 1. unify RLHF/PPO, DPO and KTO into a supervised learning of minimizing the difference between an implicit reward and an explicit reward; 2. outperform RLHF/PPO while simplify, stabilize, speed up and reduce memory burden of RL fine-tuning process; 3. accommodate different feedback types including pairwise, binary and scalar feedback. Downstream experiments show UNA outperforms DPO, KTO and RLHF.",
      "arxiv_id": "2408.15339v3",
      "published": "2024-08-27",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2408.15339v3",
      "pdf_url": "https://arxiv.org/pdf/2408.15339v3.pdf"
    },
    {
      "title": "Reinforcement Learning from Human Feedback without Reward Inference: Model-Free Algorithm and Instance-Dependent Analysis",
      "authors": [
        "Qining Zhang",
        "Honghao Wei",
        "Lei Ying"
      ],
      "summary": "In this paper, we study reinforcement learning from human feedback (RLHF) under an episodic Markov decision process with a general trajectory-wise reward model. We developed a model-free RLHF best policy identification algorithm, called $\\mathsf{BSAD}$, without explicit reward model inference, which is a critical intermediate step in the contemporary RLHF paradigms for training large language models (LLM). The algorithm identifies the optimal policy directly from human preference information in a backward manner, employing a dueling bandit sub-routine that constantly duels actions to identify the superior one. $\\mathsf{BSAD}$ adopts a reward-free exploration and best-arm-identification-like adaptive stopping criteria to equalize the visitation among all states in the same decision step while moving to the previous step as soon as the optimal action is identifiable, leading to a provable, instance-dependent sample complexity $\\tilde{\\mathcal{O}}(c_{\\mathcal{M}}SA^3H^3M\\log\\frac{1}\u03b4)$ which resembles the result in classic RL, where $c_{\\mathcal{M}}$ is the instance-dependent constant and $M$ is the batch size. Moreover, $\\mathsf{BSAD}$ can be transformed into an explore-then-commit algorithm with logarithmic regret and generalized to discounted MDPs using a frame-based approach. Our results show: (i) sample-complexity-wise, RLHF is not significantly harder than classic RL and (ii) end-to-end RLHF may deliver improved performance by avoiding pitfalls in reward inferring such as overfit and distribution shift.",
      "arxiv_id": "2406.07455v2",
      "published": "2024-06-11",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2406.07455v2",
      "pdf_url": "https://arxiv.org/pdf/2406.07455v2.pdf"
    },
    {
      "title": "Corruption Robust Offline Reinforcement Learning with Human Feedback",
      "authors": [
        "Debmalya Mandal",
        "Andi Nika",
        "Parameswaran Kamalaruban",
        "Adish Singla",
        "Goran Radanovi\u0107"
      ],
      "summary": "We study data corruption robustness for reinforcement learning with human feedback (RLHF) in an offline setting. Given an offline dataset of pairs of trajectories along with feedback about human preferences, an $\\varepsilon$-fraction of the pairs is corrupted (e.g., feedback flipped or trajectory features manipulated), capturing an adversarial attack or noisy human preferences. We aim to design algorithms that identify a near-optimal policy from the corrupted data, with provable guarantees. Existing theoretical works have separately studied the settings of corruption robust RL (learning from scalar rewards directly under corruption) and offline RLHF (learning from human feedback without corruption); however, they are inapplicable to our problem of dealing with corrupted data in offline RLHF setting. To this end, we design novel corruption robust offline RLHF methods under various assumptions on the coverage of the data-generating distributions. At a high level, our methodology robustifies an offline RLHF framework by first learning a reward model along with confidence sets and then learning a pessimistic optimal policy over the confidence set. Our key insight is that learning optimal policy can be done by leveraging an offline corruption-robust RL oracle in different ways (e.g., zero-order oracle or first-order oracle), depending on the data coverage assumptions. To our knowledge, ours is the first work that provides provable corruption robust offline RLHF methods.",
      "arxiv_id": "2402.06734v1",
      "published": "2024-02-09",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2402.06734v1",
      "pdf_url": "https://arxiv.org/pdf/2402.06734v1.pdf"
    },
    {
      "title": "Provable Reinforcement Learning from Human Feedback with an Unknown Link Function",
      "authors": [
        "Qining Zhang",
        "Lei Ying"
      ],
      "summary": "Link functions, which characterize how human preferences are generated from the value function of an RL problem, are a crucial component in designing RLHF algorithms. Almost all RLHF algorithms, including state-of-the-art ones in empirical studies such as DPO and PPO, assume the link function is known to the agent (e.g., a logistic function according to the Bradley-Terry model), which is arguably unrealistic considering the complex nature of human preferences. To avoid link function mis-specification, this paper studies general RLHF problems with unknown link functions. We propose a novel policy optimization algorithm called ZSPO based on a new zeroth-order policy optimization method, where the key is to use human preference to construct a parameter update direction that is positively correlated with the true policy gradient direction. ZSPO achieves it by estimating the sign of the value function difference instead of estimating the gradient from the value function difference, so it does not require knowing the link function. Under mild conditions, ZSPO converges to a stationary policy with a polynomial convergence rate depending on the number of policy iterations and trajectories per iteration. Numerical results also show the superiority of ZSPO under link function mismatch.",
      "arxiv_id": "2506.03066v1",
      "published": "2025-06-03",
      "categories": [
        "cs.LG",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2506.03066v1",
      "pdf_url": "https://arxiv.org/pdf/2506.03066v1.pdf"
    },
    {
      "title": "Distributionally Robust Reinforcement Learning with Human Feedback",
      "authors": [
        "Debmalya Mandal",
        "Paulius Sasnauskas",
        "Goran Radanovic"
      ],
      "summary": "Reinforcement learning from human feedback (RLHF) has evolved to be one of the main methods for fine-tuning large language models (LLMs). However, existing RLHF methods are non-robust, and their performance deteriorates if the downstream task differs significantly from the preference dataset used in fine-tuning. In order to mitigate this problem, we introduce a distributionally robust RLHF for fine-tuning LLMs. In particular, our goal is to ensure that a fine-tuned model retains its performance even when the distribution of prompts significantly differs from the distribution encountered during fine-tuning. We formulate distributionally robust optimization (DRO) version of two popular fine-tuning methods -- (1) reward-based RLHF and (2) reward-free DPO (direct preference optimization). We propose a minibatch gradient descent based algorithms for both of them, and theoretically prove convergence guarantees for the algorithms. Subsequently, we evaluate our algorithms on an out-of-distribution (OOD) task by first training the model on the Unified-Feedback dataset and evaluating its performance on two different datasets. The experimental results show that our robust training improves the accuracy of the learned reward models on average, and markedly on some tasks, such as reasoning. Furthermore, we show that the robust versions of policy optimization methods, similarly improve performance on OOD tasks.",
      "arxiv_id": "2503.00539v1",
      "published": "2025-03-01",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2503.00539v1",
      "pdf_url": "https://arxiv.org/pdf/2503.00539v1.pdf"
    },
    {
      "title": "On the Algorithmic Bias of Aligning Large Language Models with RLHF: Preference Collapse and Matching Regularization",
      "authors": [
        "Jiancong Xiao",
        "Ziniu Li",
        "Xingyu Xie",
        "Emily Getzen",
        "Cong Fang",
        "Qi Long",
        "Weijie J. Su"
      ],
      "summary": "Accurately aligning large language models (LLMs) with human preferences is crucial for informing fair, economically sound, and statistically efficient decision-making processes. However, we argue that the predominant approach for aligning LLMs with human preferences through a reward model -- reinforcement learning from human feedback (RLHF) -- suffers from an inherent algorithmic bias due to its Kullback--Leibler-based regularization in optimization. In extreme cases, this bias could lead to a phenomenon we term preference collapse, where minority preferences are virtually disregarded. To mitigate this algorithmic bias, we introduce preference matching (PM) RLHF, a novel approach that provably aligns LLMs with the preference distribution of the reward model under the Bradley--Terry--Luce/Plackett--Luce model. Central to our approach is a PM regularizer that takes the form of the negative logarithm of the LLM's policy probability distribution over responses, which helps the LLM balance response diversification and reward maximization. Notably, we obtain this regularizer by solving an ordinary differential equation that is necessary for the PM property. For practical implementation, we introduce a conditional variant of PM RLHF that is tailored to natural language generation. Finally, we empirically validate the effectiveness of conditional PM RLHF through experiments on the OPT and Llama-family models, demonstrating a 29% to 41% improvement in alignment with human preferences, as measured by a certain metric, compared to standard RLHF.",
      "arxiv_id": "2405.16455v2",
      "published": "2024-05-26",
      "categories": [
        "stat.ML",
        "cs.LG",
        "stat.ME"
      ],
      "url": "https://arxiv.org/abs/2405.16455v2",
      "pdf_url": "https://arxiv.org/pdf/2405.16455v2.pdf"
    },
    {
      "title": "The Impact of Preference Agreement in Reinforcement Learning from Human Feedback: A Case Study in Summarization",
      "authors": [
        "Sian Gooding",
        "Hassan Mansoor"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) can be used to capture complex and nuanced properties of text generation quality. As a result, the task of text summarization has been identified as a good candidate for this process. In this paper, we explore how preference agreement impacts the efficacy of RLHF for summarization. We show that sampling human preferences to include a range of annotator agreement results in (1) higher accuracy reward models and (2) alters the characteristics of quality captured. We additionally show improvements in downstream generation when using a reward model trained with a range of preference agreements. Our contributions have implications for the design of synthetic datasets as well as the importance of considering quality differentials in comparison-based data.",
      "arxiv_id": "2311.04919v1",
      "published": "2023-11-02",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.HC"
      ],
      "url": "https://arxiv.org/abs/2311.04919v1",
      "pdf_url": "https://arxiv.org/pdf/2311.04919v1.pdf"
    },
    {
      "title": "Evaluating Defences against Unsafe Feedback in RLHF",
      "authors": [
        "Domenic Rosati",
        "Giles Edkins",
        "Harsh Raj",
        "David Atanasov",
        "Subhabrata Majumdar",
        "Janarthanan Rajendran",
        "Frank Rudzicz",
        "Hassan Sajjad"
      ],
      "summary": "While there has been progress towards aligning Large Language Models (LLMs) with human values and ensuring safe behaviour at inference time, safety guards can easily be removed when fine tuned on unsafe and harmful datasets. While this setting has been treated extensively, another popular training paradigm, learning from unsafe feedback with reinforcement learning, has previously been unexplored. This is concerning due to the widespread deployment of feedback collection systems. We address this gap by providing an analysis of learning settings where feedback is harmful, i.e. that unsafe samples are preferred over safe ones despite model developers goal to maintain safety. We find that safety-aligned LLMs easily explore unsafe action spaces via generating harmful text and optimize for reward that violates safety constraints indicating that current safety guards are not enough to prevent learning from unsafe feedback. In order to protect against this vulnerability, we adapt a number of both \"implict\" and \"explicit\" harmful fine-tuning defences to evaluate whether they are effective as learning constraints in an RLHF setting finding that no method is generally effective pointing to the need for more defence research. We end the paper with the observation that some defences work by performing \"harmless reward hacking\" for which we provide a theoretical explanation drawn from the theory of Constrained Markov Decision Processes and provide some direction for future defence development.",
      "arxiv_id": "2409.12914v3",
      "published": "2024-09-19",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2409.12914v3",
      "pdf_url": "https://arxiv.org/pdf/2409.12914v3.pdf"
    },
    {
      "title": "COPR: Continual Human Preference Learning via Optimal Policy Regularization",
      "authors": [
        "Han Zhang",
        "Lin Gui",
        "Yu Lei",
        "Yuanzhao Zhai",
        "Yehong Zhang",
        "Yulan He",
        "Hui Wang",
        "Yue Yu",
        "Kam-Fai Wong",
        "Bin Liang",
        "Ruifeng Xu"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) is commonly utilized to improve the alignment of Large Language Models (LLMs) with human preferences. Given the evolving nature of human preferences, continual alignment becomes more crucial and practical in comparison to traditional static alignment. Nevertheless, making RLHF compatible with Continual Learning (CL) is challenging due to its complex process. Meanwhile, directly learning new human preferences may lead to Catastrophic Forgetting (CF) of historical preferences, resulting in helpless or harmful outputs. To overcome these challenges, we propose the Continual Optimal Policy Regularization (COPR) method, which draws inspiration from the optimal policy theory. COPR utilizes a sampling distribution as a demonstration and regularization constraints for CL. It adopts the Lagrangian Duality (LD) method to dynamically regularize the current policy based on the historically optimal policy, which prevents CF and avoids over-emphasizing unbalanced objectives. We also provide formal proof for the learnability of COPR. The experimental results show that COPR outperforms strong CL baselines on our proposed benchmark, in terms of reward-based, GPT-4 evaluations and human assessment. Furthermore, we validate the robustness of COPR under various CL settings, including different backbones, replay memory sizes, and learning orders.",
      "arxiv_id": "2402.14228v3",
      "published": "2024-02-22",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2402.14228v3",
      "pdf_url": "https://arxiv.org/pdf/2402.14228v3.pdf"
    },
    {
      "title": "Direct Preference Optimization with Unobserved Preference Heterogeneity: The Necessity of Ternary Preferences",
      "authors": [
        "Keertana Chidambaram",
        "Karthik Vinary Seetharaman",
        "Vasilis Syrgkanis"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has become central to aligning large language models with human values, typically by first learning a reward model from preference data which is then used to update the model with reinforcement learning. Recent alternatives such as Direct Preference Optimization (DPO) simplify this pipeline by directly optimizing on preferences. However, both approaches often assume uniform annotator preferences and rely on binary comparisons, overlooking two key limitations: the diversity of human evaluators and the limitations of pairwise feedback. In this work, we address both these issues. First, we connect preference learning in RLHF with the econometrics literature and show that binary comparisons are insufficient for identifying latent user preferences from finite user data and infinite users, while (even incomplete) rankings over three or more responses ensure identifiability. Second, we introduce methods to incorporate heterogeneous preferences into alignment algorithms. We develop an Expectation-Maximization adaptation of DPO that discovers latent annotator types and trains a mixture of LLMs accordingly. Then we propose an aggregation algorithm using a min-max regret fairness criterion to produce a single generative policy with equitable performance guarantees. Together, these contributions establish a theoretical and algorithmic framework for fairness and personalization for diverse users in generative model alignment.",
      "arxiv_id": "2510.15716v1",
      "published": "2025-10-17",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.15716v1",
      "pdf_url": "https://arxiv.org/pdf/2510.15716v1.pdf"
    },
    {
      "title": "Best Policy Learning from Trajectory Preference Feedback",
      "authors": [
        "Akhil Agnihotri",
        "Rahul Jain",
        "Deepak Ramachandran",
        "Zheng Wen"
      ],
      "summary": "Reinforcement Learning from Human Feedback (RLHF) has emerged as a powerful approach for aligning generative models, but its reliance on learned reward models makes it vulnerable to mis-specification and reward hacking. Preference-based Reinforcement Learning (PbRL) offers a more robust alternative by directly leveraging noisy binary comparisons over trajectories. We study the best policy identification problem in PbRL, motivated by post-training optimization of generative models, for example, during multi-turn interactions. Learning in this setting combines an offline preference dataset--potentially biased or out-of-distribution and collected from a rater of subpar 'competence'--with online pure exploration, making systematic online learning essential. To this end, we propose Posterior Sampling for Preference Learning ($\\mathsf{PSPL}$), a novel algorithm inspired by Top-Two Thompson Sampling that maintains posteriors over the reward model and dynamics. We provide the first Bayesian simple regret guarantees for PbRL and introduce an efficient approximation that outperforms existing baselines on simulation and image generation benchmarks.",
      "arxiv_id": "2501.18873v3",
      "published": "2025-01-31",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2501.18873v3",
      "pdf_url": "https://arxiv.org/pdf/2501.18873v3.pdf"
    }
  ],
  "code_synthesis": [
    {
      "title": "Logic-Q: Improving Deep Reinforcement Learning-based Quantitative Trading via Program Sketch-based Tuning",
      "authors": [
        "Zhiming Li",
        "Junzhe Jiang",
        "Yushi Cao",
        "Aixin Cui",
        "Bozhi Wu",
        "Bo Li",
        "Yang Liu",
        "Danny Dongning Sun"
      ],
      "summary": "Deep reinforcement learning (DRL) has revolutionized quantitative trading (Q-trading) by achieving decent performance without significant human expert knowledge. Despite its achievements, we observe that the current state-of-the-art DRL models are still ineffective in identifying the market trends, causing them to miss good trading opportunities or suffer from large drawdowns when encountering market crashes. To address this limitation, a natural approach is to incorporate human expert knowledge in identifying market trends. Whereas, such knowledge is abstract and hard to be quantified. In order to effectively leverage abstract human expert knowledge, in this paper, we propose a universal logic-guided deep reinforcement learning framework for Q-trading, called Logic-Q. In particular, Logic-Q adopts the program synthesis by sketching paradigm and introduces a logic-guided model design that leverages a lightweight, plug-and-play market trend-aware program sketch to determine the market trend and correspondingly adjusts the DRL policy in a post-hoc manner. Extensive evaluations of two popular quantitative trading tasks demonstrate that Logic-Q can significantly improve the performance of previous state-of-the-art DRL trading strategies.",
      "arxiv_id": "2310.05551v3",
      "published": "2023-10-09",
      "categories": [
        "cs.CE",
        "cs.AI",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2310.05551v3",
      "pdf_url": "https://arxiv.org/pdf/2310.05551v3.pdf"
    },
    {
      "title": "Learning of Generalizable and Interpretable Knowledge in Grid-Based Reinforcement Learning Environments",
      "authors": [
        "Manuel Eberhardinger",
        "Johannes Maucher",
        "Setareh Maghsudi"
      ],
      "summary": "Understanding the interactions of agents trained with deep reinforcement learning is crucial for deploying agents in games or the real world. In the former, unreasonable actions confuse players. In the latter, that effect is even more significant, as unexpected behavior cause accidents with potentially grave and long-lasting consequences for the involved individuals. In this work, we propose using program synthesis to imitate reinforcement learning policies after seeing a trajectory of the action sequence. Programs have the advantage that they are inherently interpretable and verifiable for correctness. We adapt the state-of-the-art program synthesis system DreamCoder for learning concepts in grid-based environments, specifically, a navigation task and two miniature versions of Atari games, Space Invaders and Asterix. By inspecting the generated libraries, we can make inferences about the concepts the black-box agent has learned and better understand the agent's behavior. We achieve the same by visualizing the agent's decision-making process for the imitated sequences. We evaluate our approach with different types of program synthesizers based on a search-only method, a neural-guided search, and a language model fine-tuned on code.",
      "arxiv_id": "2309.03651v1",
      "published": "2023-09-07",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2309.03651v1",
      "pdf_url": "https://arxiv.org/pdf/2309.03651v1.pdf"
    },
    {
      "title": "Reinforcement learning with learned gadgets to tackle hard quantum problems on real hardware",
      "authors": [
        "Akash Kundu",
        "Leopoldo Sarra"
      ],
      "summary": "Designing quantum circuits for specific tasks is challenging due to the exponential growth of the state space. We introduce gadget reinforcement learning (GRL), which integrates reinforcement learning with program synthesis to automatically generate and incorporate composite gates (gadgets) into the action space. This enhances the exploration of parameterized quantum circuits (PQCs) for complex tasks like approximating ground states of quantum Hamiltonians, an NP-hard problem. We evaluate GRL using the transverse field Ising model under typical computational budgets (e.g., 2- 3 days of GPU runtime). Our results show improved accuracy, hardware compatibility and scalability. GRL exhibits robust performance as the size and complexity of the problem increases, even with constrained computational resources. By integrating gadget extraction, GRL facilitates the discovery of reusable circuit components tailored for specific hardware, bridging the gap between algorithmic design and practical implementation. This makes GRL a versatile framework for optimizing quantum circuits with applications in hardware-specific optimizations and variational quantum algorithms. The code is available at: https://github.com/Aqasch/Gadget_RL",
      "arxiv_id": "2411.00230v2",
      "published": "2024-10-31",
      "categories": [
        "quant-ph",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2411.00230v2",
      "pdf_url": "https://arxiv.org/pdf/2411.00230v2.pdf"
    },
    {
      "title": "Neural Task Synthesis for Visual Programming",
      "authors": [
        "Victor-Alexandru P\u0103durean",
        "Georgios Tzannetos",
        "Adish Singla"
      ],
      "summary": "Generative neural models hold great promise in enhancing programming education by synthesizing new content. We seek to design neural models that can automatically generate programming tasks for a given specification in the context of visual programming domains. Despite the recent successes of large generative models like GPT-4, our initial results show that these models are ineffective in synthesizing visual programming tasks and struggle with logical and spatial reasoning. We propose a novel neuro-symbolic technique, NeurTaskSyn, that can synthesize programming tasks for a specification given in the form of desired programming concepts exercised by its solution code and constraints on the visual task. NeurTaskSyn has two components: the first component is trained via imitation learning procedure to generate possible solution codes, and the second component is trained via reinforcement learning procedure to guide an underlying symbolic execution engine that generates visual tasks for these codes. We demonstrate the effectiveness of NeurTaskSyn through an extensive empirical evaluation and a qualitative study on reference tasks taken from the Hour of Code: Classic Maze challenge by Code-dot-org and the Intro to Programming with Karel course by CodeHS-dot-com.",
      "arxiv_id": "2305.18342v3",
      "published": "2023-05-26",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.CY",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2305.18342v3",
      "pdf_url": "https://arxiv.org/pdf/2305.18342v3.pdf"
    },
    {
      "title": "Program Machine Policy: Addressing Long-Horizon Tasks by Integrating Program Synthesis and State Machines",
      "authors": [
        "Yu-An Lin",
        "Chen-Tao Lee",
        "Guan-Ting Liu",
        "Pu-Jen Cheng",
        "Shao-Hua Sun"
      ],
      "summary": "Deep reinforcement learning (deep RL) excels in various domains but lacks generalizability and interpretability. On the other hand, programmatic RL methods (Trivedi et al., 2021; Liu et al., 2023) reformulate RL tasks as synthesizing interpretable programs that can be executed in the environments. Despite encouraging results, these methods are limited to short-horizon tasks. On the other hand, representing RL policies using state machines (Inala et al., 2020) can inductively generalize to long-horizon tasks; however, it struggles to scale up to acquire diverse and complex behaviors. This work proposes the Program Machine Policy (POMP), which bridges the advantages of programmatic RL and state machine policies, allowing for the representation of complex behaviors and the address of long-term tasks. Specifically, we introduce a method that can retrieve a set of effective, diverse, and compatible programs. Then, we use these programs as modes of a state machine and learn a transition function to transition among mode programs, allowing for capturing repetitive behaviors. Our proposed framework outperforms programmatic RL and deep RL baselines on various tasks and demonstrates the ability to inductively generalize to even longer horizons without any fine-tuning. Ablation studies justify the effectiveness of our proposed search algorithm for retrieving a set of programs as modes.",
      "arxiv_id": "2311.15960v2",
      "published": "2023-11-27",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.PL",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2311.15960v2",
      "pdf_url": "https://arxiv.org/pdf/2311.15960v2.pdf"
    },
    {
      "title": "Continuous-Time Reinforcement Learning: New Design Algorithms with Theoretical Insights and Performance Guarantees",
      "authors": [
        "Brent A. Wallace",
        "Jennie Si"
      ],
      "summary": "Continuous-time nonlinear optimal control problems hold great promise in real-world applications. After decades of development, reinforcement learning (RL) has achieved some of the greatest successes as a general nonlinear control design method. However, a recent comprehensive analysis of state-of-the-art continuous-time RL (CT-RL) methods, namely, adaptive dynamic programming (ADP)-based CT-RL algorithms, reveals they face significant design challenges due to their complexity, numerical conditioning, and dimensional scaling issues. Despite advanced theoretical results, existing ADP CT-RL synthesis methods are inadequate in solving even small, academic problems. The goal of this work is thus to introduce a suite of new CT-RL algorithms for control of affine nonlinear systems. Our design approach relies on two important factors. First, our methods are applicable to physical systems that can be partitioned into smaller subproblems. This constructive consideration results in reduced dimensionality and greatly improved intuitiveness of design. Second, we introduce a new excitation framework to improve persistence of excitation (PE) and numerical conditioning performance via classical input/output insights. Such a design-centric approach is the first of its kind in the ADP CT-RL community. In this paper, we progressively introduce a suite of (decentralized) excitable integral reinforcement learning (EIRL) algorithms. We provide convergence and closed-loop stability guarantees, and we demonstrate these guarantees on a significant application problem of controlling an unstable, nonminimum phase hypersonic vehicle (HSV).",
      "arxiv_id": "2307.08920v1",
      "published": "2023-07-18",
      "categories": [
        "eess.SY",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2307.08920v1",
      "pdf_url": "https://arxiv.org/pdf/2307.08920v1.pdf"
    },
    {
      "title": "SATURN: SAT-based Reinforcement Learning to Unleash Language Model Reasoning",
      "authors": [
        "Huanyu Liu",
        "Jia Li",
        "Hao Zhu",
        "Kechi Zhang",
        "Yihong Dong",
        "Ge Li"
      ],
      "summary": "How to design reinforcement learning (RL) tasks that effectively unleash the reasoning capability of large language models (LLMs) remains an open question. Existing RL tasks (e.g., math, programming, and constructing reasoning tasks) suffer from three key limitations: (1) Scalability. They rely heavily on human annotation or expensive LLM synthesis to generate sufficient training data. (2) Verifiability. LLMs' outputs are hard to verify automatically and reliably. (3) Controllable Difficulty. Most tasks lack fine-grained difficulty control, making it hard to train LLMs to develop reasoning ability from easy to hard.   To address these limitations, we propose Saturn, a SAT-based RL framework that uses Boolean Satisfiability (SAT) problems to train and evaluate LLMs reasoning. Saturn enables scalable task construction, rule-based verification, and precise difficulty control. Saturn designs a curriculum learning pipeline that continuously improves LLMs' reasoning capability by constructing SAT tasks of increasing difficulty and training LLMs from easy to hard. To ensure stable training, we design a principled mechanism to control difficulty transitions.   We introduce Saturn-2.6k, a dataset of 2,660 SAT problems with varying difficulty. It supports the evaluation of how LLM reasoning changes with problem difficulty. We apply Saturn to DeepSeek-R1-Distill-Qwen and obtain Saturn-1.5B and Saturn-7B. We achieve several notable results: (1) On SAT problems, Saturn-1.5B and Saturn-7B achieve average pass@3 improvements of +14.0 and +28.1, respectively. (2) On math and programming tasks, Saturn-1.5B and Saturn-7B improve average scores by +4.9 and +1.8 on benchmarks (e.g., AIME, LiveCodeBench). (3) Compared to the state-of-the-art (SOTA) approach in constructing RL tasks, Saturn achieves further improvements of +8.8%. We release the source code, data, and models to support future research.",
      "arxiv_id": "2505.16368v2",
      "published": "2025-05-22",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.16368v2",
      "pdf_url": "https://arxiv.org/pdf/2505.16368v2.pdf"
    },
    {
      "title": "RLEF: Grounding Code LLMs in Execution Feedback with Reinforcement Learning",
      "authors": [
        "Jonas Gehring",
        "Kunhao Zheng",
        "Jade Copet",
        "Vegard Mella",
        "Quentin Carbonneaux",
        "Taco Cohen",
        "Gabriel Synnaeve"
      ],
      "summary": "Large language models (LLMs) deployed as agents solve user-specified tasks over multiple steps while keeping the required manual engagement to a minimum. Crucially, such LLMs need to ground their generations in any feedback obtained to reliably achieve the desired outcomes. We propose an end-to-end reinforcement learning method for teaching models to leverage execution feedback in the realm of code synthesis, where state-of-the-art LLMs struggle to improve code iteratively compared to independent sampling. We benchmark on competitive programming tasks, where we achieve new state-of-the art results with both small (8B parameters) and large (70B) models while reducing the amount of samples required by an order of magnitude. Our analysis of inference-time behavior demonstrates that our method produces LLMs that effectively leverage automatic feedback over multiple steps.",
      "arxiv_id": "2410.02089v2",
      "published": "2024-10-02",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2410.02089v2",
      "pdf_url": "https://arxiv.org/pdf/2410.02089v2.pdf"
    },
    {
      "title": "Language-Model-Assisted Bi-Level Programming for Reward Learning from Internet Videos",
      "authors": [
        "Harsh Mahesheka",
        "Zhixian Xie",
        "Zhaoran Wang",
        "Wanxin Jin"
      ],
      "summary": "Learning from Demonstrations, particularly from biological experts like humans and animals, often encounters significant data acquisition challenges. While recent approaches leverage internet videos for learning, they require complex, task-specific pipelines to extract and retarget motion data for the agent. In this work, we introduce a language-model-assisted bi-level programming framework that enables a reinforcement learning agent to directly learn its reward from internet videos, bypassing dedicated data preparation. The framework includes two levels: an upper level where a vision-language model (VLM) provides feedback by comparing the learner's behavior with expert videos, and a lower level where a large language model (LLM) translates this feedback into reward updates. The VLM and LLM collaborate within this bi-level framework, using a \"chain rule\" approach to derive a valid search direction for reward learning. We validate the method for reward learning from YouTube videos, and the results have shown that the proposed method enables efficient reward design from expert videos of biological agents for complex behavior synthesis.",
      "arxiv_id": "2410.09286v1",
      "published": "2024-10-11",
      "categories": [
        "cs.RO",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2410.09286v1",
      "pdf_url": "https://arxiv.org/pdf/2410.09286v1.pdf"
    },
    {
      "title": "In Search of Trees: Decision-Tree Policy Synthesis for Black-Box Systems via Search",
      "authors": [
        "Emir Demirovi\u0107",
        "Christian Schilling",
        "Anna Lukina"
      ],
      "summary": "Decision trees, owing to their interpretability, are attractive as control policies for (dynamical) systems. Unfortunately, constructing, or synthesising, such policies is a challenging task. Previous approaches do so by imitating a neural-network policy, approximating a tabular policy obtained via formal synthesis, employing reinforcement learning, or modelling the problem as a mixed-integer linear program. However, these works may require access to a hard-to-obtain accurate policy or a formal model of the environment (within reach of formal synthesis), and may not provide guarantees on the quality or size of the final tree policy. In contrast, we present an approach to synthesise optimal decision-tree policies given a deterministic black-box environment and specification, a discretisation of the tree predicates, and an initial set of states, where optimality is defined with respect to the number of steps to achieve the goal. Our approach is a specialised search algorithm which systematically explores the (exponentially large) space of decision trees under the given discretisation. The key component is a novel trace-based pruning mechanism that significantly reduces the search space. Our approach represents a conceptually novel way of synthesising small decision-tree policies with optimality guarantees even for black-box environments with black-box specifications.",
      "arxiv_id": "2409.03260v2",
      "published": "2024-09-05",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2409.03260v2",
      "pdf_url": "https://arxiv.org/pdf/2409.03260v2.pdf"
    },
    {
      "title": "UNSAT Solver Synthesis via Monte Carlo Forest Search",
      "authors": [
        "Chris Cameron",
        "Jason Hartford",
        "Taylor Lundy",
        "Tuan Truong",
        "Alan Milligan",
        "Rex Chen",
        "Kevin Leyton-Brown"
      ],
      "summary": "We introduce Monte Carlo Forest Search (MCFS), a class of reinforcement learning (RL) algorithms for learning policies in {tree MDPs}, for which policy execution involves traversing an exponential-sized tree. Examples of such problems include proving unsatisfiability of a SAT formula; counting the number of solutions of a satisfiable SAT formula; and finding the optimal solution to a mixed-integer program. MCFS algorithms can be seen as extensions of Monte Carlo Tree Search (MCTS) to cases where, rather than finding a good path (solution) within a tree, the problem is to find a small tree within a forest of candidate trees. We instantiate and evaluate our ideas in an algorithm that we dub Knuth Synthesis, an MCFS algorithm that learns DPLL branching policies for solving the Boolean satisfiability (SAT) problem, with the objective of achieving good average-case performance on a given distribution of unsatisfiable problem instances. Knuth Synthesis is the first RL approach to avoid the prohibitive costs of policy evaluations in an exponentially-sized tree, leveraging two key ideas: first, we estimate tree size by randomly sampling paths and measuring their lengths, drawing on an unbiased approximation due to Knuth (1975); second, we query a strong solver at a user-defined depth rather than learning a policy across the whole tree, to focus our policy search on early decisions that offer the greatest potential for reducing tree size. We matched or exceeded the performance of a strong baseline on three well-known SAT distributions, facing problems that were two orders of magnitude more challenging than those addressed in previous RL studies.",
      "arxiv_id": "2211.12581v3",
      "published": "2022-11-22",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2211.12581v3",
      "pdf_url": "https://arxiv.org/pdf/2211.12581v3.pdf"
    },
    {
      "title": "Executable Functional Abstractions: Inferring Generative Programs for Advanced Math Problems",
      "authors": [
        "Zaid Khan",
        "Elias Stengel-Eskin",
        "Archiki Prasad",
        "Jaemin Cho",
        "Mohit Bansal"
      ],
      "summary": "Scientists often infer abstract procedures from specific instances of problems and use the abstractions to generate new, related instances. For example, programs encoding the formal rules and properties of a system have been useful in fields ranging from reinforcement learning (procedural environments) to physics (simulation engines). These programs can be seen as functions which execute to different outputs based on their parameterizations (e.g., gridworld configuration or initial physical conditions). We introduce the term EFA (Executable Functional Abstraction) to denote such programs for math problems. EFA-like constructs have been shown to be useful for mathematical reasoning as problem generators for stress-testing models. However, prior work has been limited to automatically constructing abstractions for grade-school math (whose simple rules are easy to encode in programs), while generating EFAs for advanced math has thus far required human engineering. We explore the automatic construction of EFAs for advanced mathematics problems by developing EFAGen, which operationalizes the task of automatically inferring an EFA for a given seed problem and solution as a program synthesis task. We first formalize the properties of any valid EFA as executable unit tests. Using execution feedback from the unit tests, we search over candidate programs sampled from a LLM to find EFA programs that are faithful to the generalized problem and solution class underlying the seed problem. We then apply the tests as a reward signal, training LLMs to become better writers of EFAs. We show that EFAs inferred by EFAGen are faithful to the seed problems, produce learnable problem variations, and that EFAGen can infer EFAs across diverse sources of competition-level math problems. Finally, we show uses of model-written EFAs e.g., finding harder/easier problem variants, as well as data generation.",
      "arxiv_id": "2504.09763v2",
      "published": "2025-04-14",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.09763v2",
      "pdf_url": "https://arxiv.org/pdf/2504.09763v2.pdf"
    },
    {
      "title": "Exploring RL-based LLM Training for Formal Language Tasks with Programmed Rewards",
      "authors": [
        "Alexander G. Padula",
        "Dennis J. N. J. Soemers"
      ],
      "summary": "Proximal Policy Optimization (PPO) is commonly used in Reinforcement Learning from Human Feedback to align large language models (LLMs) with downstream tasks. This paper investigates the feasibility of using PPO for direct reinforcement learning (RL) from explicitly programmed reward signals, as opposed to indirect learning from human feedback via an intermediary reward model. We focus on tasks expressed through formal languages, such as mathematics and programming, where explicit reward functions can be programmed to automatically assess the quality of generated outputs. We apply this approach to a sentiment alignment task, a simple arithmetic task, and a more complex game synthesis task. The sentiment alignment task replicates prior research and serves to validate our experimental setup. Our results show that pure RL-based training for the two formal language tasks is challenging, with success being limited even for the simple arithmetic task. We propose a novel batch-entropy regularization term to aid exploration, although training is not yet entirely stable. Our findings suggest that direct RL training of LLMs may be more suitable for relatively minor changes, such as alignment, than for learning new tasks altogether, even if an informative reward signal can be expressed programmatically.",
      "arxiv_id": "2410.17126v1",
      "published": "2024-10-22",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.17126v1",
      "pdf_url": "https://arxiv.org/pdf/2410.17126v1.pdf"
    },
    {
      "title": "ACECODER: Acing Coder RL via Automated Test-Case Synthesis",
      "authors": [
        "Huaye Zeng",
        "Dongfu Jiang",
        "Haozhe Wang",
        "Ping Nie",
        "Xiaotong Chen",
        "Wenhu Chen"
      ],
      "summary": "Most progress in recent coder models has been driven by supervised fine-tuning (SFT), while the potential of reinforcement learning (RL) remains largely unexplored, primarily due to the lack of reliable reward data/model in the code domain. In this paper, we address this challenge by leveraging automated large-scale test-case synthesis to enhance code model training. Specifically, we design a pipeline that generates extensive (question, test-cases) pairs from existing code data. Using these test cases, we construct preference pairs based on pass rates over sampled programs to train reward models with Bradley-Terry loss. It shows an average of 10-point improvement for Llama-3.1-8B-Ins and 5-point improvement for Qwen2.5-Coder-7B-Ins through best-of-32 sampling, making the 7B model on par with 236B DeepSeek-V2.5. Furthermore, we conduct reinforcement learning with both reward models and test-case pass rewards, leading to consistent improvements across HumanEval, MBPP, BigCodeBench, and LiveCodeBench (V4). Notably, we follow the R1-style training to start from Qwen2.5-Coder-base directly and show that our RL training can improve model on HumanEval-plus by over 25\\% and MBPP-plus by 6\\% for merely 80 optimization steps. We believe our results highlight the huge potential of reinforcement learning in coder models.",
      "arxiv_id": "2502.01718v4",
      "published": "2025-02-03",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2502.01718v4",
      "pdf_url": "https://arxiv.org/pdf/2502.01718v4.pdf"
    },
    {
      "title": "Synthesizing world models for bilevel planning",
      "authors": [
        "Zergham Ahmed",
        "Joshua B. Tenenbaum",
        "Christopher J. Bates",
        "Samuel J. Gershman"
      ],
      "summary": "Modern reinforcement learning (RL) systems have demonstrated remarkable capabilities in complex environments, such as video games. However, they still fall short of achieving human-like sample efficiency and adaptability when learning new domains. Theory-based reinforcement learning (TBRL) is an algorithmic framework specifically designed to address this gap. Modeled on cognitive theories, TBRL leverages structured, causal world models - \"theories\" - as forward simulators for use in planning, generalization and exploration. Although current TBRL systems provide compelling explanations of how humans learn to play video games, they face several technical limitations: their theory languages are restrictive, and their planning algorithms are not scalable. To address these challenges, we introduce TheoryCoder, an instantiation of TBRL that exploits hierarchical representations of theories and efficient program synthesis methods for more powerful learning and planning. TheoryCoder equips agents with general-purpose abstractions (e.g., \"move to\"), which are then grounded in a particular environment by learning a low-level transition model (a Python program synthesized from observations by a large language model). A bilevel planning algorithm can exploit this hierarchical structure to solve large domains. We demonstrate that this approach can be successfully applied to diverse and challenging grid-world games, where approaches based on directly synthesizing a policy perform poorly. Ablation studies demonstrate the benefits of using hierarchical abstractions.",
      "arxiv_id": "2503.20124v2",
      "published": "2025-03-26",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.20124v2",
      "pdf_url": "https://arxiv.org/pdf/2503.20124v2.pdf"
    },
    {
      "title": "RLSF: Fine-tuning LLMs via Symbolic Feedback",
      "authors": [
        "Piyush Jha",
        "Prithwish Jana",
        "Pranavkrishna Suresh",
        "Arnav Arora",
        "Vijay Ganesh"
      ],
      "summary": "Large Language Models (LLMs) have transformed AI but often struggle with tasks that require domain-specific reasoning and logical alignment. Traditional fine-tuning methods do not leverage the vast amount of symbolic domain-knowledge available to us via symbolic reasoning tools (e.g., provers), and are further limited by sparse rewards and unreliable reward models.   We introduce Reinforcement Learning via Symbolic Feedback (RLSF), a novel fine-tuning paradigm where symbolic reasoning tools (e.g., solvers, provers, and algebra systems) provide fine-grained feedback to LLMs. RLSF uses poly-sized certificates (e.g., proofs) generated by symbolic tools to identify and correct errors in model outputs, offering token-level guidance without requiring differentiable reasoning systems. This paradigm bridges the gap between symbolic reasoning and LLM fine-tuning, enabling precise alignment with domain-specific constraints while addressing key limitations of traditional reward signals.   Via extensive evaluations, we show that our RLSF-based fine-tuning of LLMs outperforms traditional approaches on five different applications (that have some associated logical or domain constraints), namely, program synthesis from natural language pseudo-code to programming language, three chemistry tasks, and solving the Game of 24. A key takeaway is that fine-tuning via RLSF enables relatively smaller LLMs to significantly outperform closed-source models that are orders of magnitude larger.",
      "arxiv_id": "2405.16661v3",
      "published": "2024-05-26",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG",
        "cs.LO"
      ],
      "url": "https://arxiv.org/abs/2405.16661v3",
      "pdf_url": "https://arxiv.org/pdf/2405.16661v3.pdf"
    },
    {
      "title": "TritonRL: Training LLMs to Think and Code Triton Without Cheating",
      "authors": [
        "Jiin Woo",
        "Shaowei Zhu",
        "Allen Nie",
        "Zhen Jia",
        "Yida Wang",
        "Youngsuk Park"
      ],
      "summary": "With the rapid evolution of large language models (LLMs), the demand for automated, high-performance system kernels has emerged as a key enabler for accelerating development and deployment. We introduce TritonRL, a domain-specialized LLM for Triton kernel generation, trained with a novel training framework that enables robust and automated kernel synthesis. Unlike general-purpose programming languages, Triton kernel generation faces unique challenges due to data scarcity and incomplete evaluation criteria, vulnerable to reward hacking. Our approach addresses these challenges end-to-end by distilling Triton-specific knowledge through supervised fine-tuning on curated datasets, and further improving code quality via reinforcement learning (RL) with robust, verifiable rewards and hierarchical reward assignment. Our RL framework robustly detects reward hacking and guides both reasoning traces and code tokens through fine-grained verification and hierarchical reward decomposition, enabling the model to generate high-quality Triton kernels that can truly replace existing modules. With robust and fine-grained evaluation, our experiments on KernelBench demonstrate that TritonRL achieves state-of-the-art correctness and speedup, surpassing all other Triton-specific models and underscoring the effectiveness of our RL-based training paradigm.",
      "arxiv_id": "2510.17891v1",
      "published": "2025-10-18",
      "categories": [
        "cs.SE",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.17891v1",
      "pdf_url": "https://arxiv.org/pdf/2510.17891v1.pdf"
    },
    {
      "title": "QiMeng-CodeV-R1: Reasoning-Enhanced Verilog Generation",
      "authors": [
        "Yaoyu Zhu",
        "Di Huang",
        "Hanqi Lyu",
        "Xiaoyun Zhang",
        "Chongxiao Li",
        "Wenxuan Shi",
        "Yutong Wu",
        "Jianan Mu",
        "Jinghua Wang",
        "Yang Zhao",
        "Pengwei Jin",
        "Shuyao Cheng",
        "Shengwen Liang",
        "Xishan Zhang",
        "Rui Zhang",
        "Zidong Du",
        "Qi Guo",
        "Xing Hu",
        "Yunji Chen"
      ],
      "summary": "Large language models (LLMs) trained via reinforcement learning with verifiable reward (RLVR) have achieved breakthroughs on tasks with explicit, automatable verification, such as software programming and mathematical problems. Extending RLVR to electronic design automation (EDA), especially automatically generating hardware description languages (HDLs) like Verilog from natural-language (NL) specifications, however, poses three key challenges: the lack of automated and accurate verification environments, the scarcity of high-quality NL-code pairs, and the prohibitive computation cost of RLVR. To this end, we introduce CodeV-R1, an RLVR framework for training Verilog generation LLMs. First, we develop a rule-based testbench generator that performs robust equivalence checking against golden references. Second, we propose a round-trip data synthesis method that pairs open-source Verilog snippets with LLM-generated NL descriptions, verifies code-NL-code consistency via the generated testbench, and filters out inequivalent examples to yield a high-quality dataset. Third, we employ a two-stage \"distill-then-RL\" training pipeline: distillation for the cold start of reasoning abilities, followed by adaptive DAPO, our novel RLVR algorithm that can reduce training cost by adaptively adjusting sampling rate. The resulting model, CodeV-R1-7B, achieves 68.6% and 72.9% pass@1 on VerilogEval v2 and RTLLM v1.1, respectively, surpassing prior state-of-the-art by 12~20%, while even exceeding the performance of 671B DeepSeek-R1 on RTLLM. We have released our model, training code, and dataset to facilitate research in EDA and LLM communities.",
      "arxiv_id": "2505.24183v4",
      "published": "2025-05-30",
      "categories": [
        "cs.LG",
        "cs.AR",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2505.24183v4",
      "pdf_url": "https://arxiv.org/pdf/2505.24183v4.pdf"
    },
    {
      "title": "LLMs for Relational Reasoning: How Far are We?",
      "authors": [
        "Zhiming Li",
        "Yushi Cao",
        "Xiufeng Xu",
        "Junzhe Jiang",
        "Xu Liu",
        "Yon Shin Teo",
        "Shang-wei Lin",
        "Yang Liu"
      ],
      "summary": "Large language models (LLMs) have revolutionized many areas (e.g. natural language processing, software engineering, etc.) by achieving state-of-the-art performance on extensive downstream tasks. Aiming to achieve robust and general artificial intelligence, there has been a surge of interest in investigating the reasoning ability of the LLMs. Whereas the textual and numerical reasoning benchmarks adopted by previous works are rather shallow and simple, it is hard to conclude that the LLMs possess strong reasoning ability by merely achieving positive results on these benchmarks. Recent efforts have demonstrated that the LLMs are poor at solving sequential decision-making problems that require common-sense planning by evaluating their performance on the reinforcement learning benchmarks. In this work, we conduct an in-depth assessment of several state-of-the-art LLMs' reasoning ability based on the inductive logic programming (ILP) benchmark, which is broadly recognized as a representative and challenging measurement for evaluating logic program induction/synthesis systems as it requires inducing strict cause-effect logic to achieve robust deduction on independent and identically distributed (IID) and out-of-distribution (OOD) test samples. Our evaluations illustrate that compared with the neural program induction systems which are much smaller in model size, the state-of-the-art LLMs are much poorer in terms of reasoning ability by achieving much lower performance and generalization using either natural language prompting or truth-value matrix prompting.",
      "arxiv_id": "2401.09042v1",
      "published": "2024-01-17",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2401.09042v1",
      "pdf_url": "https://arxiv.org/pdf/2401.09042v1.pdf"
    },
    {
      "title": "Code-Driven Planning in Grid Worlds with Large Language Models",
      "authors": [
        "Ashwath Vaithinathan Aravindan",
        "Zhisheng Tang",
        "Mayank Kejriwal"
      ],
      "summary": "We propose an iterative programmatic planning (IPP) framework for solving grid-based tasks by synthesizing interpretable agent policies expressed in code using large language models (LLMs). Instead of relying on traditional search or reinforcement learning, our approach uses code generation as policy synthesis, where the LLM outputs executable programs that map environment states to action sequences. Our proposed architecture incorporates several prompting strategies, including direct code generation, pseudocode-conditioned refinement, and curriculum-based prompting, but also includes an iterative refinement mechanism that updates code based on task performance feedback. We evaluate our approach using six leading LLMs and two challenging grid-based benchmarks (GRASP and MiniGrid). Our IPP framework demonstrates improvements over direct code generation ranging from 10\\% to as much as 10x across five of the six models and establishes a new state-of-the-art result for GRASP. IPP is found to significantly outperform direct elicitation of a solution from GPT-o3-mini (by 63\\% on MiniGrid to 116\\% on GRASP), demonstrating the viability of the overall approach. Computational costs of all code generation approaches are similar. While code generation has a higher initial prompting cost compared to direct solution elicitation (\\$0.08 per task vs. \\$0.002 per instance for GPT-o3-mini), the code can be reused for any number of instances, making the amortized cost significantly lower (by 400x on GPT-o3-mini across the complete GRASP benchmark).",
      "arxiv_id": "2505.10749v1",
      "published": "2025-05-15",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.10749v1",
      "pdf_url": "https://arxiv.org/pdf/2505.10749v1.pdf"
    }
  ],
  "llm_reasoning": [
    {
      "title": "SwS: Self-aware Weakness-driven Problem Synthesis in Reinforcement Learning for LLM Reasoning",
      "authors": [
        "Xiao Liang",
        "Zhong-Zhi Li",
        "Yeyun Gong",
        "Yang Wang",
        "Hengyuan Zhang",
        "Yelong Shen",
        "Ying Nian Wu",
        "Weizhu Chen"
      ],
      "summary": "Reinforcement Learning with Verifiable Rewards (RLVR) has proven effective for training large language models (LLMs) on complex reasoning tasks, such as mathematical problem solving. A prerequisite for the scalability of RLVR is a high-quality problem set with precise and verifiable answers. However, the scarcity of well-crafted human-labeled math problems and limited-verification answers in existing distillation-oriented synthetic datasets limit their effectiveness in RL. Additionally, most problem synthesis strategies indiscriminately expand the problem set without considering the model's capabilities, leading to low efficiency in generating useful questions. To mitigate this issue, we introduce a Self-aware Weakness-driven problem Synthesis framework (SwS) that systematically identifies model deficiencies and leverages them for problem augmentation. Specifically, we define weaknesses as questions that the model consistently fails to learn through its iterative sampling during RL training. We then extract the core concepts from these failure cases and synthesize new problems to strengthen the model's weak areas in subsequent augmented training, enabling it to focus on and gradually overcome its weaknesses. Without relying on external knowledge distillation, our framework enables robust generalization byempowering the model to self-identify and address its weaknesses in RL, yielding average performance gains of 10.0% and 7.7% on 7B and 32B models across eight mainstream reasoning benchmarks.",
      "arxiv_id": "2506.08989v1",
      "published": "2025-06-10",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.08989v1",
      "pdf_url": "https://arxiv.org/pdf/2506.08989v1.pdf"
    },
    {
      "title": "Large Language Models Reasoning Abilities Under Non-Ideal Conditions After RL-Fine-Tuning",
      "authors": [
        "Chang Tian",
        "Matthew B. Blaschko",
        "Mingzhe Xing",
        "Xiuxing Li",
        "Yinliang Yue",
        "Marie-Francine Moens"
      ],
      "summary": "Reinforcement learning (RL) has become a key technique for enhancing the reasoning abilities of large language models (LLMs), with policy-gradient algorithms dominating the post-training stage because of their efficiency and effectiveness. However, most existing benchmarks evaluate large-language-model reasoning under idealized settings, overlooking performance in realistic, non-ideal scenarios. We identify three representative non-ideal scenarios with practical relevance: summary inference, fine-grained noise suppression, and contextual filtering. We introduce a new research direction guided by brain-science findings that human reasoning remains reliable under imperfect inputs. We formally define and evaluate these challenging scenarios. We fine-tune three LLMs and a state-of-the-art large vision-language model (LVLM) using RL with a representative policy-gradient algorithm and then test their performance on eight public datasets. Our results reveal that while RL fine-tuning improves baseline reasoning under idealized settings, performance declines significantly across all three non-ideal scenarios, exposing critical limitations in advanced reasoning capabilities. Although we propose a scenario-specific remediation method, our results suggest current methods leave these reasoning deficits largely unresolved. This work highlights that the reasoning abilities of large models are often overstated and underscores the importance of evaluating models under non-ideal scenarios. The code and data will be released at XXXX.",
      "arxiv_id": "2508.04848v1",
      "published": "2025-08-06",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.04848v1",
      "pdf_url": "https://arxiv.org/pdf/2508.04848v1.pdf"
    },
    {
      "title": "Chain-of-Thought Hub: A Continuous Effort to Measure Large Language Models' Reasoning Performance",
      "authors": [
        "Yao Fu",
        "Litu Ou",
        "Mingyu Chen",
        "Yuhao Wan",
        "Hao Peng",
        "Tushar Khot"
      ],
      "summary": "As large language models (LLMs) are continuously being developed, their evaluation becomes increasingly important yet challenging. This work proposes Chain-of-Thought Hub, an open-source evaluation suite on the multi-step reasoning capabilities of large language models. We are interested in this setting for two reasons: (1) from the behavior of GPT and PaLM model family, we observe that complex reasoning is likely to be a key differentiator between weaker and stronger LLMs; (2) we envisage large language models to become the next-generation computational platform and foster an ecosystem of LLM-based new applications, this naturally requires the foundation models to perform complex tasks that often involve the composition of linguistic and logical operations. Our approach is to compile a suite of challenging reasoning benchmarks to track the progress of LLMs. Our current results show that: (1) model scale clearly correlates with reasoning capabilities; (2) As of May 2023, Claude-v1.3 and PaLM-2 are the only two models that are comparable with GPT-4, while open-sourced models still lag behind; (3) LLaMA-65B performs closely to code-davinci-002, indicating that with successful further development such as reinforcement learning from human feedback (RLHF), it has great potential to be close to GPT-3.5-Turbo. Our results also suggest that for the open-source efforts to catch up, the community may focus more on building better base models and exploring RLHF.",
      "arxiv_id": "2305.17306v1",
      "published": "2023-05-26",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2305.17306v1",
      "pdf_url": "https://arxiv.org/pdf/2305.17306v1.pdf"
    },
    {
      "title": "Reinforcement Learning Meets Large Language Models: A Survey of Advancements and Applications Across the LLM Lifecycle",
      "authors": [
        "Keliang Liu",
        "Dingkang Yang",
        "Ziyun Qian",
        "Weijie Yin",
        "Yuchi Wang",
        "Hongsheng Li",
        "Jun Liu",
        "Peng Zhai",
        "Yang Liu",
        "Lihua Zhang"
      ],
      "summary": "In recent years, training methods centered on Reinforcement Learning (RL) have markedly enhanced the reasoning and alignment performance of Large Language Models (LLMs), particularly in understanding human intents, following user instructions, and bolstering inferential strength. Although existing surveys offer overviews of RL augmented LLMs, their scope is often limited, failing to provide a comprehensive summary of how RL operates across the full lifecycle of LLMs. We systematically review the theoretical and practical advancements whereby RL empowers LLMs, especially Reinforcement Learning with Verifiable Rewards (RLVR). First, we briefly introduce the basic theory of RL. Second, we thoroughly detail application strategies for RL across various phases of the LLM lifecycle, including pre-training, alignment fine-tuning, and reinforced reasoning. In particular, we emphasize that RL methods in the reinforced reasoning phase serve as a pivotal driving force for advancing model reasoning to its limits. Next, we collate existing datasets and evaluation benchmarks currently used for RL fine-tuning, spanning human-annotated datasets, AI-assisted preference data, and program-verification-style corpora. Subsequently, we review the mainstream open-source tools and training frameworks available, providing clear practical references for subsequent research. Finally, we analyse the future challenges and trends in the field of RL-enhanced LLMs. This survey aims to present researchers and practitioners with the latest developments and frontier trends at the intersection of RL and LLMs, with the goal of fostering the evolution of LLMs that are more intelligent, generalizable, and secure.",
      "arxiv_id": "2509.16679v1",
      "published": "2025-09-20",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.16679v1",
      "pdf_url": "https://arxiv.org/pdf/2509.16679v1.pdf"
    },
    {
      "title": "Learning Adaptive Parallel Reasoning with Language Models",
      "authors": [
        "Jiayi Pan",
        "Xiuyu Li",
        "Long Lian",
        "Charlie Snell",
        "Yifei Zhou",
        "Adam Yala",
        "Trevor Darrell",
        "Kurt Keutzer",
        "Alane Suhr"
      ],
      "summary": "Scaling inference-time computation has substantially improved the reasoning capabilities of language models. However, existing methods have significant limitations: serialized chain-of-thought approaches generate overly long outputs, leading to increased latency and exhausted context windows, while parallel methods such as self-consistency suffer from insufficient coordination, resulting in redundant computations and limited performance gains. To address these shortcomings, we propose Adaptive Parallel Reasoning (APR), a novel reasoning framework that enables language models to orchestrate both serialized and parallel computations end-to-end. APR generalizes existing reasoning methods by enabling adaptive multi-threaded inference using spawn() and join() operations. A key innovation is our end-to-end reinforcement learning strategy, optimizing both parent and child inference threads to enhance task success rate without requiring predefined reasoning structures. Experiments on the Countdown reasoning task demonstrate significant benefits of APR: (1) higher performance within the same context window (83.4% vs. 60.0% at 4k context); (2) superior scalability with increased computation (80.1% vs. 66.6% at 20k total tokens); (3) improved accuracy at equivalent latency (75.2% vs. 57.3% at approximately 5,000ms). APR represents a step towards enabling language models to autonomously optimize their reasoning processes through adaptive allocation of computation.",
      "arxiv_id": "2504.15466v2",
      "published": "2025-04-21",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.15466v2",
      "pdf_url": "https://arxiv.org/pdf/2504.15466v2.pdf"
    },
    {
      "title": "Enhancing Spatial Reasoning in Vision-Language Models via Chain-of-Thought Prompting and Reinforcement Learning",
      "authors": [
        "Binbin Ji",
        "Siddharth Agrawal",
        "Qiance Tang",
        "Yvonne Wu"
      ],
      "summary": "This study investigates the spatial reasoning capabilities of vision-language models (VLMs) through Chain-of-Thought (CoT) prompting and reinforcement learning. We begin by evaluating the impact of different prompting strategies and find that simple CoT formats, where the model generates a reasoning step before the answer, not only fail to help, but can even harm the model's original performance. In contrast, structured multi-stage prompting based on scene graphs (SceneGraph CoT) significantly improves spatial reasoning accuracy. Furthermore, to improve spatial reasoning ability, we fine-tune models using Group Relative Policy Optimization (GRPO) on the SAT dataset and evaluate their performance on CVBench. Compared to supervised fine-tuning (SFT), GRPO achieves higher accuracy on Pass@1 evaluations and demonstrates superior robustness under out-of-distribution (OOD) conditions. In particular, we find that SFT overfits to surface-level linguistic patterns and may degrade performance when test-time phrasing changes (e.g., from \"closer to\" to \"farther from\"). GRPO, on the other hand, generalizes more reliably and maintains stable performance under such shifts. Our findings provide insights into how reinforcement learning and structured prompting improve the spatial reasoning capabilities and generalization behavior of modern VLMs. All code is open source at: https://github.com/Yvonne511/spatial-vlm-investigator",
      "arxiv_id": "2507.13362v1",
      "published": "2025-07-06",
      "categories": [
        "cs.CV",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2507.13362v1",
      "pdf_url": "https://arxiv.org/pdf/2507.13362v1.pdf"
    },
    {
      "title": "Collaborating with language models for embodied reasoning",
      "authors": [
        "Ishita Dasgupta",
        "Christine Kaeser-Chen",
        "Kenneth Marino",
        "Arun Ahuja",
        "Sheila Babayan",
        "Felix Hill",
        "Rob Fergus"
      ],
      "summary": "Reasoning in a complex and ambiguous environment is a key goal for Reinforcement Learning (RL) agents. While some sophisticated RL agents can successfully solve difficult tasks, they require a large amount of training data and often struggle to generalize to new unseen environments and new tasks. On the other hand, Large Scale Language Models (LSLMs) have exhibited strong reasoning ability and the ability to to adapt to new tasks through in-context learning. However, LSLMs do not inherently have the ability to interrogate or intervene on the environment. In this work, we investigate how to combine these complementary abilities in a single system consisting of three parts: a Planner, an Actor, and a Reporter. The Planner is a pre-trained language model that can issue commands to a simple embodied agent (the Actor), while the Reporter communicates with the Planner to inform its next command. We present a set of tasks that require reasoning, test this system's ability to generalize zero-shot and investigate failure cases, and demonstrate how components of this system can be trained with reinforcement-learning to improve performance.",
      "arxiv_id": "2302.00763v1",
      "published": "2023-02-01",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2302.00763v1",
      "pdf_url": "https://arxiv.org/pdf/2302.00763v1.pdf"
    },
    {
      "title": "RL-STaR: Theoretical Analysis of Reinforcement Learning Frameworks for Self-Taught Reasoner",
      "authors": [
        "Fu-Chieh Chang",
        "Yu-Ting Lee",
        "Hui-Ying Shih",
        "Yi Hsuan Tseng",
        "Pei-Yuan Wu"
      ],
      "summary": "The reasoning abilities of large language models (LLMs) have improved with chain-of-thought (CoT) prompting, allowing models to solve complex tasks stepwise. However, training CoT capabilities requires detailed reasoning data, which is often scarce. The self-taught reasoner (STaR) framework addresses this by using reinforcement learning to automatically generate reasoning steps, reducing reliance on human-labeled data. Although STaR and its variants have demonstrated empirical success, a theoretical foundation explaining these improvements is lacking. This work provides a theoretical framework for understanding the effectiveness of reinforcement learning on CoT reasoning and STaR. Our contributions are: (1) criteria for the quality of pre-trained models necessary to initiate effective reasoning improvement; (2) an analysis of policy improvement, showing why LLM reasoning improves iteratively with STaR; (3) conditions for convergence to an optimal reasoning policy; and (4) an examination of STaR's robustness, explaining how it can improve reasoning even when incorporating occasional incorrect steps; This framework aims to bridge empirical findings with theoretical insights, advancing reinforcement learning approaches for reasoning in LLMs.",
      "arxiv_id": "2410.23912v2",
      "published": "2024-10-31",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.23912v2",
      "pdf_url": "https://arxiv.org/pdf/2410.23912v2.pdf"
    },
    {
      "title": "Reinforcing the Diffusion Chain of Lateral Thought with Diffusion Language Models",
      "authors": [
        "Zemin Huang",
        "Zhiyang Chen",
        "Zijun Wang",
        "Tiancheng Li",
        "Guo-Jun Qi"
      ],
      "summary": "We introduce the Diffusion Chain of Lateral Thought (DCoLT), a reasoning framework for diffusion language models. DCoLT treats each intermediate step in the reverse diffusion process as a latent \"thinking\" action and optimizes the entire reasoning trajectory to maximize the reward on the correctness of the final answer with outcome-based Reinforcement Learning (RL). Unlike traditional Chain-of-Thought (CoT) methods that follow a causal, linear thinking process, DCoLT allows bidirectional, non-linear reasoning with no strict rule on grammatical correctness amid its intermediate steps of thought. We implement DCoLT on two representative Diffusion Language Models (DLMs). First, we choose SEDD as a representative continuous-time discrete diffusion model, where its concrete score derives a probabilistic policy to maximize the RL reward over the entire sequence of intermediate diffusion steps. We further consider the discrete-time masked diffusion language model -- LLaDA, and find that the order to predict and unmask tokens plays an essential role to optimize its RL action resulting from the ranking-based Unmasking Policy Module (UPM) defined by the Plackett-Luce model. Experiments on both math and code generation tasks show that using only public data and 16 H800 GPUs, DCoLT-reinforced DLMs outperform other DLMs trained by SFT or RL or even both. Notably, DCoLT-reinforced LLaDA boosts its reasoning accuracy by +9.8%, +5.7%, +11.4%, +19.5% on GSM8K, MATH, MBPP, and HumanEval.",
      "arxiv_id": "2505.10446v3",
      "published": "2025-05-15",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.10446v3",
      "pdf_url": "https://arxiv.org/pdf/2505.10446v3.pdf"
    },
    {
      "title": "RoiRL: Efficient, Self-Supervised Reasoning with Offline Iterative Reinforcement Learning",
      "authors": [
        "Aleksei Arzhantsev",
        "Otmane Sakhi",
        "Flavian Vasile"
      ],
      "summary": "Reinforcement learning (RL) is central to improving reasoning in large language models (LLMs) but typically requires ground-truth rewards. Test-Time Reinforcement Learning (TTRL) removes this need by using majority-vote rewards, but relies on heavy online RL and incurs substantial computational cost. We propose RoiRL: Reasoning with offline iterative Reinforcement Learning, a family of lightweight offline learning alternatives that can target the same regularized optimal policies. Unlike TTRL, RoiRL eliminates the need to maintain a reference model and instead optimizes weighted log-likelihood objectives, enabling stable training with significantly lower memory and compute requirements. Experimental results show that RoiRL trains to 2.5x faster and consistently outperforms TTRL on reasoning benchmarks, establishing a scalable path to self-improving LLMs without labels.",
      "arxiv_id": "2510.02892v1",
      "published": "2025-10-03",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.02892v1",
      "pdf_url": "https://arxiv.org/pdf/2510.02892v1.pdf"
    },
    {
      "title": "Improve Vision Language Model Chain-of-thought Reasoning",
      "authors": [
        "Ruohong Zhang",
        "Bowen Zhang",
        "Yanghao Li",
        "Haotian Zhang",
        "Zhiqing Sun",
        "Zhe Gan",
        "Yinfei Yang",
        "Ruoming Pang",
        "Yiming Yang"
      ],
      "summary": "Chain-of-thought (CoT) reasoning in vision language models (VLMs) is crucial for improving interpretability and trustworthiness. However, current training recipes lack robust CoT reasoning data, relying on datasets dominated by short annotations with minimal rationales. In this work, we show that training VLM on short answers does not generalize well to reasoning tasks that require more detailed responses. To address this, we propose a two-fold approach. First, we distill rationales from GPT-4o model to enrich the training data and fine-tune VLMs, boosting their CoT performance. Second, we apply reinforcement learning to further calibrate reasoning quality. Specifically, we construct positive (correct) and negative (incorrect) pairs of model-generated reasoning chains, by comparing their predictions with annotated short answers. Using this pairwise data, we apply the Direct Preference Optimization algorithm to refine the model's reasoning abilities. Our experiments demonstrate significant improvements in CoT reasoning on benchmark datasets and better generalization to direct answer prediction as well. This work emphasizes the importance of incorporating detailed rationales in training and leveraging reinforcement learning to strengthen the reasoning capabilities of VLMs.",
      "arxiv_id": "2410.16198v1",
      "published": "2024-10-21",
      "categories": [
        "cs.AI",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2410.16198v1",
      "pdf_url": "https://arxiv.org/pdf/2410.16198v1.pdf"
    },
    {
      "title": "Eliciting Chain-of-Thought Reasoning for Time Series Analysis using Reinforcement Learning",
      "authors": [
        "Felix Parker",
        "Nimeesha Chan",
        "Chi Zhang",
        "Kimia Ghobadi"
      ],
      "summary": "Complex numerical time series analysis often demands multi-step reasoning capabilities beyond current models' reach. Tasks like medical diagnosis and weather forecasting require sequential reasoning processes -- including counterfactual analysis, logical deduction, knowledge application, and multi-modal contextual integration -- that existing time series models cannot explicitly perform. While recent research has shown large language models (LLMs) can achieve sophisticated Chain-of-Thought (CoT) reasoning through reinforcement learning (RL), these advances have primarily focused on mathematical and coding domains, with LLMs still demonstrating poor performance on time series tasks. We introduce Chain Of thought for Understanding Numerical Time Series (COUNTS), the first framework that trains LLMs to perform CoT reasoning across diverse time series tasks using RL with verifiable rewards. Our approach employs a Residual Vector-Quantized VAE to create high-fidelity discrete tokens that seamlessly integrate into a pre-trained LLM's vocabulary. COUNTS undergoes a two-stage training process: first, supervised fine-tuning on time series analysis tasks to master our novel representations, followed by Group Relative Policy Optimization training on verifiable problems using prompting strategies that encourage explicit reasoning steps before producing final answers. Our experiments demonstrate that this RL-driven approach with intermediate CoT reasoning significantly enhances LLM performance across various time series analysis tasks, opening new possibilities for complex temporal data reasoning.",
      "arxiv_id": "2510.01116v1",
      "published": "2025-10-01",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.01116v1",
      "pdf_url": "https://arxiv.org/pdf/2510.01116v1.pdf"
    },
    {
      "title": "Reason Like a Radiologist: Chain-of-Thought and Reinforcement Learning for Verifiable Report Generation",
      "authors": [
        "Peiyuan Jing",
        "Kinhei Lee",
        "Zhenxuan Zhang",
        "Huichi Zhou",
        "Zhengqing Yuan",
        "Zhifan Gao",
        "Lei Zhu",
        "Giorgos Papanastasiou",
        "Yingying Fang",
        "Guang Yang"
      ],
      "summary": "Radiology report generation is critical for efficiency but current models lack the structured reasoning of experts, hindering clinical trust and explainability by failing to link visual findings to precise anatomical locations. This paper introduces BoxMed-RL, a groundbreaking unified training framework for generating spatially verifiable and explainable radiology reports. Built on a large vision-language model, BoxMed-RL revolutionizes report generation through two integrated phases: (1) In the Pretraining Phase, we refine the model via medical concept learning, using Chain-of-Thought supervision to internalize the radiologist-like workflow, followed by spatially verifiable reinforcement, which applies reinforcement learning to align medical findings with bounding boxes. (2) In the Downstream Adapter Phase, we freeze the pretrained weights and train a downstream adapter to ensure fluent and clinically credible reports. This framework precisely mimics radiologists' workflow, compelling the model to connect high-level medical concepts with definitive anatomical evidence. Extensive experiments on public datasets demonstrate that BoxMed-RL achieves an average 7% improvement in both METEOR and ROUGE-L metrics compared to state-of-the-art methods. An average 5% improvement in large language model-based metrics further underscores BoxMed-RL's robustness in generating high-quality radiology reports.",
      "arxiv_id": "2504.18453v1",
      "published": "2025-04-25",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.18453v1",
      "pdf_url": "https://arxiv.org/pdf/2504.18453v1.pdf"
    },
    {
      "title": "T1: Advancing Language Model Reasoning through Reinforcement Learning and Inference Scaling",
      "authors": [
        "Zhenyu Hou",
        "Xin Lv",
        "Rui Lu",
        "Jiajie Zhang",
        "Yujiang Li",
        "Zijun Yao",
        "Juanzi Li",
        "Jie Tang",
        "Yuxiao Dong"
      ],
      "summary": "Large language models (LLMs) have demonstrated remarkable capabilities in complex reasoning tasks. However, existing approaches mainly rely on imitation learning and struggle to achieve effective test-time scaling. While reinforcement learning (RL) holds promise for enabling self-exploration, recent attempts yield modest improvements in complex reasoning. In this paper, we present T1 to scale RL by encouraging exploration and understand inference scaling. We first initialize the LLM using synthesized chain-of-thought data that integrates trial-and-error and self-verification. To scale RL training, we promote increased sampling diversity through oversampling. We demonstrate that T1 with open LLMs as its base exhibits inference scaling behavior and achieves superior performance on challenging math reasoning benchmarks. More importantly, we present a simple strategy to examine inference scaling, where increased inference budgets directly lead to T1's better performance without any additional verification.",
      "arxiv_id": "2501.11651v2",
      "published": "2025-01-20",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.11651v2",
      "pdf_url": "https://arxiv.org/pdf/2501.11651v2.pdf"
    },
    {
      "title": "Interleaved Reasoning for Large Language Models via Reinforcement Learning",
      "authors": [
        "Roy Xie",
        "David Qiu",
        "Deepak Gopinath",
        "Dong Lin",
        "Yanchao Sun",
        "Chong Wang",
        "Saloni Potdar",
        "Bhuwan Dhingra"
      ],
      "summary": "Long chain-of-thought (CoT) significantly enhances large language models' (LLM) reasoning capabilities. However, the extensive reasoning traces lead to inefficiencies and an increased time-to-first-token (TTFT). We propose a novel training paradigm that uses reinforcement learning (RL) to guide reasoning LLMs to interleave thinking and answering for multi-hop questions. We observe that models inherently possess the ability to perform interleaved reasoning, which can be further enhanced through RL. We introduce a simple yet effective rule-based reward to incentivize correct intermediate steps, which guides the policy model toward correct reasoning paths by leveraging intermediate signals generated during interleaved reasoning. Extensive experiments conducted across five diverse datasets and three RL algorithms (PPO, GRPO, and REINFORCE++) demonstrate consistent improvements over traditional think-answer reasoning, without requiring external tools. Specifically, our approach reduces TTFT by over 80% on average and improves up to 19.3% in Pass@1 accuracy. Furthermore, our method, trained solely on question answering and logical reasoning datasets, exhibits strong generalization ability to complex reasoning datasets such as MATH, GPQA, and MMLU. Additionally, we conduct in-depth analysis to reveal several valuable insights into conditional reward modeling.",
      "arxiv_id": "2505.19640v1",
      "published": "2025-05-26",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.19640v1",
      "pdf_url": "https://arxiv.org/pdf/2505.19640v1.pdf"
    },
    {
      "title": "Bridging Formal Language with Chain-of-Thought Reasoning to Geometry Problem Solving",
      "authors": [
        "Tianyun Yang",
        "Yunwen Li",
        "Ziniu Li",
        "Zhihang Lin",
        "Ruoyu Sun",
        "Tian Ding"
      ],
      "summary": "Large vision language models exhibit notable limitations on Geometry Problem Solving (GPS) because of their unreliable diagram interpretation and pure natural-language reasoning. A recent line of work mitigates this by using symbolic solvers: the model directly generates a formal program that a geometry solver can execute. However, this direct program generation lacks intermediate reasoning, making the decision process opaque and prone to errors. In this work, we explore a new approach that integrates Chain-of-Thought (CoT) with formal language. The model interleaves natural language reasoning with incremental emission of solver-executable code, producing a hybrid reasoning trace in which critical derivations are expressed in formal language. To teach this behavior at scale, we combine (1) supervised fine-tuning on an 11K newly developed synthetic dataset with interleaved natural language reasoning and automatic formalization, and (2) solver-in-the-loop reinforcement learning that jointly optimizes both the CoT narrative and the resulting program through outcome-based rewards. Built on Qwen2.5-VL-7B, our new model, named GF-Reasoner, achieves up to 15% accuracy improvements on standard GPS benchmarks, surpassing both 7B-scale peers and the much larger model Qwen2.5-VL-72B. By exploiting high-order geometric knowledge and offloading symbolic computation to the solver, the generated reasoning traces are noticeably shorter and cleaner. Furthermore, we present a comprehensive analysis of method design choices (e.g., reasoning paradigms, data synthesis, training epochs, etc.), providing actionable insights for future research.",
      "arxiv_id": "2508.09099v1",
      "published": "2025-08-12",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2508.09099v1",
      "pdf_url": "https://arxiv.org/pdf/2508.09099v1.pdf"
    },
    {
      "title": "Legal$\u0394$: Enhancing Legal Reasoning in LLMs via Reinforcement Learning with Chain-of-Thought Guided Information Gain",
      "authors": [
        "Xin Dai",
        "Buqiang Xu",
        "Zhenghao Liu",
        "Yukun Yan",
        "Huiyuan Xie",
        "Xiaoyuan Yi",
        "Shuo Wang",
        "Ge Yu"
      ],
      "summary": "Legal Artificial Intelligence (LegalAI) has achieved notable advances in automating judicial decision-making with the support of Large Language Models (LLMs). However, existing legal LLMs still struggle to generate reliable and interpretable reasoning processes. They often default to fast-thinking behavior by producing direct answers without explicit multi-step reasoning, limiting their effectiveness in complex legal scenarios that demand rigorous justification. To address this challenge, we propose Legal$\u0394$, a reinforcement learning framework designed to enhance legal reasoning through chain-of-thought guided information gain. During training, Legal$\u0394$ employs a dual-mode input setup-comprising direct answer and reasoning-augmented modes-and maximizes the information gain between them. This encourages the model to acquire meaningful reasoning patterns rather than generating superficial or redundant explanations. Legal$\u0394$ follows a two-stage approach: (1) distilling latent reasoning capabilities from a powerful Large Reasoning Model (LRM), DeepSeek-R1, and (2) refining reasoning quality via differential comparisons, combined with a multidimensional reward mechanism that assesses both structural coherence and legal-domain specificity. Experimental results on multiple legal reasoning tasks demonstrate that Legal$\u0394$ outperforms strong baselines in both accuracy and interpretability. It consistently produces more robust and trustworthy legal judgments without relying on labeled preference data. All code and data will be released at https://github.com/NEUIR/LegalDelta.",
      "arxiv_id": "2508.12281v2",
      "published": "2025-08-17",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2508.12281v2",
      "pdf_url": "https://arxiv.org/pdf/2508.12281v2.pdf"
    },
    {
      "title": "CARFT: Boosting LLM Reasoning via Contrastive Learning with Annotated Chain-of-Thought-based Reinforced Fine-Tuning",
      "authors": [
        "Wenqiao Zhu",
        "Ji Liu",
        "Rongjuncheng Zhang",
        "Haipang Wu",
        "Yulun Zhang"
      ],
      "summary": "Reasoning capability plays a significantly critical role in the the broad applications of Large Language Models (LLMs). To enhance the reasoning performance of LLMs, diverse Reinforcement Learning (RL)-based fine-tuning approaches have been proposed to address the limited generalization capability of LLMs trained solely via Supervised Fine-Tuning (SFT). Despite their effectiveness, two major limitations hinder the advancement of LLMs. First, vanilla RL-based approaches ignore annotated Chain-of-Thought (CoT) and incorporate unstable reasoning path sampling, which typically results in model collapse, unstable training process, and suboptimal performance. Second, existing SFT approaches generally overemphasize the annotated CoT, potentially leading to performance degradation due to insufficient exploitation of potential CoT. In this paper, we propose a Contrastive learning with annotated CoT-based Reinforced Fine-Tuning approach, i.e., \\TheName{}, to enhance the reasoning performance of LLMs while addressing the aforementioned limitations. Specifically, we propose learning a representation for each CoT. Based on this representation, we design novel contrastive signals to guide the fine-tuning process. Our approach not only fully exploits the available annotated CoT but also stabilizes the fine-tuning procedure by incorporating an additional unsupervised learning signal. We conduct comprehensive experiments and in-depth analysis with three baseline approaches, two foundation models, and two datasets to demonstrate significant advantages of \\TheName{} in terms of robustness, performance (up to 10.15\\%), and efficiency (up to 30.62\\%). Code is available at https://github.com/WNQzhu/CARFT.",
      "arxiv_id": "2508.15868v2",
      "published": "2025-08-21",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.15868v2",
      "pdf_url": "https://arxiv.org/pdf/2508.15868v2.pdf"
    },
    {
      "title": "L1: Controlling How Long A Reasoning Model Thinks With Reinforcement Learning",
      "authors": [
        "Pranjal Aggarwal",
        "Sean Welleck"
      ],
      "summary": "Reasoning language models have shown an uncanny ability to improve performance at test-time by ``thinking longer''-that is, by generating longer chain-of-thought sequences and hence using more compute. However, the length of their chain-of-thought reasoning is not controllable, making it impossible to allocate test-time compute to achieve a desired level of performance. We introduce Length Controlled Policy Optimization (LCPO), a simple reinforcement learning method that optimizes for accuracy and adherence to user-specified length constraints. We use LCPO to train L1, a reasoning language model that produces outputs satisfying a length constraint given in its prompt. L1's length control allows for smoothly trading off computational cost and accuracy on a wide range of tasks, and outperforms the state-of-the-art S1 method for length control. Furthermore, we uncover an unexpected short chain-of-thought capability in models trained with LCPO. Specifically, using LCPO we derive Short Reasoning Models (SRMs), that exhibit similar reasoning patterns as full-length reasoning models, but can generate CoT lengths comparable to non-reasoning models. They demonstrate significant performance gains, for instance, our 1.5B L1 model surpasses GPT-4o at equal reasoning lengths. Overall, LCPO enables precise control over reasoning length, allowing for fine-grained allocation of test-time compute and accuracy. We release code and models at https://www.cmu-l3.github.io/l1",
      "arxiv_id": "2503.04697v2",
      "published": "2025-03-06",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2503.04697v2",
      "pdf_url": "https://arxiv.org/pdf/2503.04697v2.pdf"
    },
    {
      "title": "Towards System 2 Reasoning in LLMs: Learning How to Think With Meta Chain-of-Thought",
      "authors": [
        "Violet Xiang",
        "Charlie Snell",
        "Kanishk Gandhi",
        "Alon Albalak",
        "Anikait Singh",
        "Chase Blagden",
        "Duy Phung",
        "Rafael Rafailov",
        "Nathan Lile",
        "Dakota Mahan",
        "Louis Castricato",
        "Jan-Philipp Franken",
        "Nick Haber",
        "Chelsea Finn"
      ],
      "summary": "We propose a novel framework, Meta Chain-of-Thought (Meta-CoT), which extends traditional Chain-of-Thought (CoT) by explicitly modeling the underlying reasoning required to arrive at a particular CoT. We present empirical evidence from state-of-the-art models exhibiting behaviors consistent with in-context search, and explore methods for producing Meta-CoT via process supervision, synthetic data generation, and search algorithms. Finally, we outline a concrete pipeline for training a model to produce Meta-CoTs, incorporating instruction tuning with linearized search traces and reinforcement learning post-training. Finally, we discuss open research questions, including scaling laws, verifier roles, and the potential for discovering novel reasoning algorithms. This work provides a theoretical and practical roadmap to enable Meta-CoT in LLMs, paving the way for more powerful and human-like reasoning in artificial intelligence.",
      "arxiv_id": "2501.04682v1",
      "published": "2025-01-08",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.04682v1",
      "pdf_url": "https://arxiv.org/pdf/2501.04682v1.pdf"
    },
    {
      "title": "Expanding Reasoning Potential in Foundation Model by Learning Diverse Chains of Thought Patterns",
      "authors": [
        "Xuemiao Zhang",
        "Can Ren",
        "Chengying Tu",
        "Rongxiang Weng",
        "Shuo Wang",
        "Hongfei Yan",
        "Jingang Wang",
        "Xunliang Cai"
      ],
      "summary": "Recent progress in large reasoning models for challenging mathematical reasoning has been driven by reinforcement learning (RL). Incorporating long chain-of-thought (CoT) data during mid-training has also been shown to substantially improve reasoning depth. However, current approaches often utilize CoT data indiscriminately, leaving open the critical question of which data types most effectively enhance model reasoning capabilities. In this paper, we define the foundation model's reasoning potential for the first time as the inverse of the number of independent attempts required to correctly answer the question, which is strongly correlated with the final model performance. We then propose utilizing diverse data enriched with high-value reasoning patterns to expand the reasoning potential. Specifically, we abstract atomic reasoning patterns from CoT sequences, characterized by commonality and inductive capabilities, and use them to construct a core reference set enriched with valuable reasoning patterns. Furthermore, we propose a dual-granularity algorithm involving chains of reasoning patterns and token entropy, efficiently selecting high-value CoT data (CoTP) from the data pool that aligns with the core set, thereby training models to master reasoning effectively. Only 10B-token CoTP data enables the 85A6B Mixture-of-Experts (MoE) model to improve by 9.58% on the challenging AIME 2024 and 2025, and to raise the upper bound of downstream RL performance by 7.81%.",
      "arxiv_id": "2509.21124v2",
      "published": "2025-09-25",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.21124v2",
      "pdf_url": "https://arxiv.org/pdf/2509.21124v2.pdf"
    },
    {
      "title": "Stop Overthinking: A Survey on Efficient Reasoning for Large Language Models",
      "authors": [
        "Yang Sui",
        "Yu-Neng Chuang",
        "Guanchu Wang",
        "Jiamu Zhang",
        "Tianyi Zhang",
        "Jiayi Yuan",
        "Hongyi Liu",
        "Andrew Wen",
        "Shaochen Zhong",
        "Na Zou",
        "Hanjie Chen",
        "Xia Hu"
      ],
      "summary": "Large Language Models (LLMs) have demonstrated remarkable capabilities in complex tasks. Recent advancements in Large Reasoning Models (LRMs), such as OpenAI o1 and DeepSeek-R1, have further improved performance in System-2 reasoning domains like mathematics and programming by harnessing supervised fine-tuning (SFT) and reinforcement learning (RL) techniques to enhance the Chain-of-Thought (CoT) reasoning. However, while longer CoT reasoning sequences improve performance, they also introduce significant computational overhead due to verbose and redundant outputs, known as the \"overthinking phenomenon\". In this paper, we provide the first structured survey to systematically investigate and explore the current progress toward achieving efficient reasoning in LLMs. Overall, relying on the inherent mechanism of LLMs, we categorize existing works into several key directions: (1) model-based efficient reasoning, which considers optimizing full-length reasoning models into more concise reasoning models or directly training efficient reasoning models; (2) reasoning output-based efficient reasoning, which aims to dynamically reduce reasoning steps and length during inference; (3) input prompts-based efficient reasoning, which seeks to enhance reasoning efficiency based on input prompt properties such as difficulty or length control. Additionally, we introduce the use of efficient data for training reasoning models, explore the reasoning capabilities of small language models, and discuss evaluation methods and benchmarking. Project website: https://github.com/Eclipsess/Awesome-Efficient-Reasoning-LLMs",
      "arxiv_id": "2503.16419v4",
      "published": "2025-03-20",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2503.16419v4",
      "pdf_url": "https://arxiv.org/pdf/2503.16419v4.pdf"
    },
    {
      "title": "Chart-R1: Chain-of-Thought Supervision and Reinforcement for Advanced Chart Reasoner",
      "authors": [
        "Lei Chen",
        "Xuanle Zhao",
        "Zhixiong Zeng",
        "Jing Huang",
        "Yufeng Zhong",
        "Lin Ma"
      ],
      "summary": "Recently, inspired by OpenAI-o1/o3 and Deepseek-R1, the R1-Style method based on reinforcement learning fine-tuning has received widespread attention from the community. Previous R1-Style methods mainly focus on mathematical reasoning and code intelligence. It is of great research significance to verify their advantages on more general multimodal data. Chart is an important multimodal data type with rich information, which brings important research challenges in complex reasoning. In this work, we introduce Chart-R1, a chart-domain vision-language model with reinforcement learning fine-tuning to enable complex chart reasoning. To support Chart-R1, we first propose a novel programmatic data synthesis technology to generate high-quality step-by-step chart reasoning data covering single- and multi-subcharts, which makes up for the lack of reasoning data in the chart domain. Then we develop a two-stage training strategy: Chart-COT with step-by-step chain-of-thought supervision, and Chart-RFT with numerically sensitive reinforcement fine-tuning. Chart-COT aims to decompose complex chart reasoning tasks into fine-grained, understandable subtasks through step-by-step supervision, which lays a good foundation for improving the reasoning level of reinforcement learning. Chart-RFT utilize the typical group relative policy optimization strategy, in which a relatively soft reward is adopted for numerical response to emphasize the numerical sensitivity in the chart domain. We conduct extensive experiments on open-source benchmarks and self-built chart reasoning dataset (\\emph{i.e., ChartRQA}). Experimental results show that Chart-R1 has significant advantages compared to chart-domain methods, even comparable to open/closed source large-scale models (\\emph{e.g., GPT-4o, Claude-3.5}).",
      "arxiv_id": "2507.15509v2",
      "published": "2025-07-21",
      "categories": [
        "cs.AI",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2507.15509v2",
      "pdf_url": "https://arxiv.org/pdf/2507.15509v2.pdf"
    },
    {
      "title": "Reasoning Under 1 Billion: Memory-Augmented Reinforcement Learning for Large Language Models",
      "authors": [
        "Hung Le",
        "Dai Do",
        "Dung Nguyen",
        "Svetha Venkatesh"
      ],
      "summary": "Recent advances in fine-tuning large language models (LLMs) with reinforcement learning (RL) have shown promising improvements in complex reasoning tasks, particularly when paired with chain-of-thought (CoT) prompting. However, these successes have been largely demonstrated on large-scale models with billions of parameters, where a strong pretraining foundation ensures effective initial exploration. In contrast, RL remains challenging for tiny LLMs with 1 billion parameters or fewer because they lack the necessary pretraining strength to explore effectively, often leading to suboptimal reasoning patterns. This work introduces a novel intrinsic motivation approach that leverages episodic memory to address this challenge, improving tiny LLMs in CoT reasoning tasks. Inspired by human memory-driven learning, our method leverages successful reasoning patterns stored in memory while allowing for controlled exploration to generate novel responses. Intrinsic rewards are computed efficiently using a kNN-based episodic memory, allowing the model to discover new reasoning strategies while quickly adapting to effective past solutions. Experiments on fine-tuning GSM8K and AI-MO datasets demonstrate that our approach significantly enhances smaller LLMs' sample efficiency and generalization capability, making RL-based reasoning improvements more accessible in low-resource settings.",
      "arxiv_id": "2504.02273v1",
      "published": "2025-04-03",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.02273v1",
      "pdf_url": "https://arxiv.org/pdf/2504.02273v1.pdf"
    },
    {
      "title": "Latent Chain-of-Thought for Visual Reasoning",
      "authors": [
        "Guohao Sun",
        "Hang Hua",
        "Jian Wang",
        "Jiebo Luo",
        "Sohail Dianat",
        "Majid Rabbani",
        "Raghuveer Rao",
        "Zhiqiang Tao"
      ],
      "summary": "Chain-of-thought (CoT) reasoning is critical for improving the interpretability and reliability of Large Vision-Language Models (LVLMs). However, existing training algorithms such as SFT, PPO, and GRPO may not generalize well across unseen reasoning tasks and heavily rely on a biased reward model. To address this challenge, we reformulate reasoning in LVLMs as posterior inference and propose a scalable training algorithm based on amortized variational inference. By leveraging diversity-seeking reinforcement learning algorithms, we introduce a novel sparse reward function for token-level learning signals that encourage diverse, high-likelihood latent CoT, overcoming deterministic sampling limitations and avoiding reward hacking. Additionally, we implement a Bayesian inference-scaling strategy that replaces costly Best-of-N and Beam Search with a marginal likelihood to efficiently rank optimal rationales and answers. We empirically demonstrate that the proposed method enhances the state-of-the-art LVLMs on seven reasoning benchmarks, in terms of effectiveness, generalization, and interpretability.",
      "arxiv_id": "2510.23925v2",
      "published": "2025-10-27",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.23925v2",
      "pdf_url": "https://arxiv.org/pdf/2510.23925v2.pdf"
    },
    {
      "title": "Unilaw-R1: A Large Language Model for Legal Reasoning with Reinforcement Learning and Iterative Inference",
      "authors": [
        "Hua Cai",
        "Shuang Zhao",
        "Liang Zhang",
        "Xuli Shen",
        "Qing Xu",
        "Weilin Shen",
        "Zihao Wen",
        "Tianke Ban"
      ],
      "summary": "Reasoning-focused large language models (LLMs) are rapidly evolving across various domains, yet their capabilities in handling complex legal problems remains underexplored. In this paper, we introduce Unilaw-R1, a large language model tailored for legal reasoning. With a lightweight 7-billion parameter scale, Unilaw-R1 significantly reduces deployment cost while effectively tackling three core challenges in the legal domain: insufficient legal knowledge, unreliable reasoning logic, and weak business generalization. To address these issues, we first construct Unilaw-R1-Data, a high-quality dataset containing 17K distilled and screened chain-of-thought (CoT) samples. Based on this, we adopt a two-stage training strategy combining Supervised Fine-Tuning (SFT) and Reinforcement Learning (RL), which significantly boosts the performance on complex legal reasoning tasks and supports interpretable decision-making in legal AI applications. To assess legal reasoning ability, we also introduce Unilaw-R1-Eval, a dedicated benchmark designed to evaluate models across single- and multi-choice legal tasks. Unilaw-R1 demonstrates strong results on authoritative benchmarks, outperforming all models of similar scale and achieving performance on par with the much larger DeepSeek-R1-Distill-Qwen-32B (54.9%). Following domain-specific training, it also showed significant gains on LawBench and LexEval, exceeding Qwen-2.5-7B-Instruct (46.6%) by an average margin of 6.6%.",
      "arxiv_id": "2510.10072v1",
      "published": "2025-10-11",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.10072v1",
      "pdf_url": "https://arxiv.org/pdf/2510.10072v1.pdf"
    },
    {
      "title": "Not All Thoughts are Generated Equal: Efficient LLM Reasoning via Multi-Turn Reinforcement Learning",
      "authors": [
        "Yansong Ning",
        "Wei Li",
        "Jun Fang",
        "Naiqiang Tan",
        "Hao Liu"
      ],
      "summary": "Compressing long chain-of-thought (CoT) from large language models (LLMs) is an emerging strategy to improve the reasoning efficiency of LLMs. Despite its promising benefits, existing studies equally compress all thoughts within a long CoT, hindering more concise and effective reasoning. To this end, we first investigate the importance of different thoughts by examining their effectiveness and efficiency in contributing to reasoning through automatic long CoT chunking and Monte Carlo rollouts. Building upon the insights, we propose a theoretically bounded metric to jointly measure the effectiveness and efficiency of different thoughts. We then propose Long$\\otimes$Short, an efficient reasoning framework that enables two LLMs to collaboratively solve the problem: a long-thought LLM for more effectively generating important thoughts, while a short-thought LLM for efficiently generating remaining thoughts. Specifically, we begin by synthesizing a small amount of cold-start data to fine-tune LLMs for long-thought and short-thought reasoning styles, respectively. Furthermore, we propose a synergizing-oriented multi-turn reinforcement learning, focusing on the model self-evolution and collaboration between long-thought and short-thought LLMs. Experimental results show that our method enables Qwen2.5-7B and Llama3.1-8B to achieve comparable performance compared to DeepSeek-R1-Distill-Qwen-7B and DeepSeek-R1-Distill-Llama-8B, while reducing token length by over 80% across the MATH500, AIME24/25, AMC23, and GPQA Diamond benchmarks. Our data and code are available at https://github.com/usail-hkust/LongShort.",
      "arxiv_id": "2505.11827v2",
      "published": "2025-05-17",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.11827v2",
      "pdf_url": "https://arxiv.org/pdf/2505.11827v2.pdf"
    },
    {
      "title": "AdaCoT: Pareto-Optimal Adaptive Chain-of-Thought Triggering via Reinforcement Learning",
      "authors": [
        "Chenwei Lou",
        "Zewei Sun",
        "Xinnian Liang",
        "Meng Qu",
        "Wei Shen",
        "Wenqi Wang",
        "Yuntao Li",
        "Qingping Yang",
        "Shuangzhi Wu"
      ],
      "summary": "Large Language Models (LLMs) have demonstrated remarkable capabilities but often face challenges with tasks requiring sophisticated reasoning. While Chain-of-Thought (CoT) prompting significantly enhances reasoning, it indiscriminately generates lengthy reasoning steps for all queries, leading to substantial computational costs and inefficiency, especially for simpler inputs. To address this critical issue, we introduce AdaCoT (Adaptive Chain-of-Thought), a novel framework enabling LLMs to adaptively decide when to invoke CoT. AdaCoT framed adaptive reasoning as a Pareto optimization problem that seeks to balance model performance with the costs associated with CoT invocation (both frequency and computational overhead). We propose a reinforcement learning (RL) based method, specifically utilizing Proximal Policy Optimization (PPO), to dynamically control the CoT triggering decision boundary by adjusting penalty coefficients, thereby allowing the model to determine CoT necessity based on implicit query complexity. A key technical contribution is Selective Loss Masking (SLM), designed to counteract decision boundary collapse during multi-stage RL training, ensuring robust and stable adaptive triggering. Experimental results demonstrate that AdaCoT successfully navigates the Pareto frontier, achieving substantial reductions in CoT usage for queries not requiring elaborate reasoning. For instance, on our production traffic testset, AdaCoT reduced CoT triggering rates to as low as 3.18\\% and decreased average response tokens by 69.06%, while maintaining high performance on complex tasks.",
      "arxiv_id": "2505.11896v2",
      "published": "2025-05-17",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.11896v2",
      "pdf_url": "https://arxiv.org/pdf/2505.11896v2.pdf"
    },
    {
      "title": "ARES: Alternating Reinforcement Learning and Supervised Fine-Tuning for Enhanced Multi-Modal Chain-of-Thought Reasoning Through Diverse AI Feedback",
      "authors": [
        "Ju-Seung Byun",
        "Jiyun Chun",
        "Jihyung Kil",
        "Andrew Perrault"
      ],
      "summary": "Large Multimodal Models (LMMs) excel at comprehending human instructions and demonstrate remarkable results across a broad spectrum of tasks. Reinforcement Learning from Human Feedback (RLHF) and AI Feedback (RLAIF) further refine LLMs by aligning them with specific preferences. These methods primarily use ranking-based feedback for entire generations. With advanced AI models (Teacher), such as GPT-4 and Claude 3 Opus, we can request various types of detailed feedback that are expensive for humans to provide. We propose a two-stage algorithm ARES that Alternates REinforcement Learning (RL) and Supervised Fine-Tuning (SFT). First, we request the Teacher to score how much each sentence contributes to solving the problem in a Chain-of-Thought (CoT). This sentence-level feedback allows us to consider individual valuable segments, providing more granular rewards for the RL procedure. Second, we ask the Teacher to correct the wrong reasoning after the RL stage. The RL procedure requires massive efforts for hyperparameter tuning and often generates errors like repetitive words and incomplete sentences. With the correction feedback, we stabilize the RL fine-tuned model through SFT. We conduct experiments on multi-model dataset ScienceQA and A-OKVQA to demonstrate the effectiveness of our proposal. ARES rationale reasoning achieves around 70% win rate against baseline models judged by GPT-4o. Additionally, we observe that the improved rationale reasoning leads to a 2.5% increase in inference answer accuracy on average for the multi-modal datasets.",
      "arxiv_id": "2407.00087v2",
      "published": "2024-06-25",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2407.00087v2",
      "pdf_url": "https://arxiv.org/pdf/2407.00087v2.pdf"
    },
    {
      "title": "Causally-Enhanced Reinforcement Policy Optimization",
      "authors": [
        "Xiangqi Wang",
        "Yue Huang",
        "Yujun Zhou",
        "Xiaonan Luo",
        "Kehan Guo",
        "Xiangliang Zhang"
      ],
      "summary": "Large language models (LLMs) trained with reinforcement objectives often achieve superficially correct answers via shortcut strategies, pairing correct outputs with spurious or unfaithful reasoning and degrading under small causal perturbations. We introduce Causally-Enhanced Policy Optimization (CE-PO), a drop-in reward-shaping framework that augments policy optimization with a differentiable proxy for causal coherence along the generation pathway from prompt (Z) to rationale (X) to answer (Y). CE-PO estimates model-internal influence with Jacobian-based sensitivities, counterfactually hardens these signals to suppress nuisance cues, and fuses the resulting coherence score with task-accuracy feedback via a Minkowski (power-mean) combiner, exposing a single tunable between accuracy and coherence trade-off. The unified reward integrates with PPO/GRPO without architectural changes. Across reasoning benchmarks and causal stress tests, CE-PO reduces reward hacking and unfaithful chain-of-thought while improving robustness to correlation-causation flips and light counterfactual edits, all at near-parity accuracy. Experimental results across 4 datasets show that CE-PO improves accuracy over baselines by 5.49% on average (up to 9.58%), while improving robustness to correlation-causation flips and light counterfactual edits.",
      "arxiv_id": "2509.23095v1",
      "published": "2025-09-27",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.23095v1",
      "pdf_url": "https://arxiv.org/pdf/2509.23095v1.pdf"
    },
    {
      "title": "Rethinking Reasoning Quality in Large Language Models through Enhanced Chain-of-Thought via RL",
      "authors": [
        "Haoyang He",
        "Zihua Rong",
        "Kun Ji",
        "Chenyang Li",
        "Qing Huang",
        "Chong Xia",
        "Lan Yang",
        "Honggang Zhang"
      ],
      "summary": "Reinforcement learning (RL) has recently become the dominant paradigm for strengthening the reasoning abilities of large language models (LLMs). Yet the rule-based reward functions commonly used on mathematical or programming benchmarks assess only answer format and correctness, providing no signal as to whether the induced Chain-of-Thought (CoT) actually improves the answer. Furthermore, such task-specific training offers limited control over logical depth and therefore may fail to reveal a model's genuine reasoning capacity. We propose Dynamic Reasoning Efficiency Reward (DRER) -- a plug-and-play RL reward framework that reshapes both reward and advantage signals. (i) A Reasoning Quality Reward assigns fine-grained credit to those reasoning chains that demonstrably raise the likelihood of the correct answer, directly incentivising the trajectories with beneficial CoT tokens. (ii) A Dynamic Length Advantage decays the advantage of responses whose length deviates from a validation-derived threshold, stabilising training. To facilitate rigorous assessment, we also release Logictree, a dynamically constructed deductive reasoning dataset that functions both as RL training data and as a comprehensive benchmark. Experiments confirm the effectiveness of DRER: our 7B model attains GPT-o3-mini level performance on Logictree with 400 trianing steps, while the average confidence of CoT-augmented answers rises by 30%. The model further exhibits generalisation across diverse logical-reasoning datasets, and the mathematical benchmark AIME24. These results illuminate how RL shapes CoT behaviour and chart a practical path toward enhancing formal-reasoning skills in large language models. All code and data are available in repository https://github.com/Henryhe09/DRER.",
      "arxiv_id": "2509.06024v1",
      "published": "2025-09-07",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2509.06024v1",
      "pdf_url": "https://arxiv.org/pdf/2509.06024v1.pdf"
    },
    {
      "title": "Reasoning Models Sometimes Output Illegible Chains of Thought",
      "authors": [
        "Arun Jose"
      ],
      "summary": "Language models trained via outcome-based reinforcement learning (RL) to reason using chain-of-thought (CoT) have shown remarkable performance. Monitoring such a model's CoT may allow us to understand its intentions and detect potential malicious behavior. However, to be effective, this requires that CoTs are legible and faithful. We study CoT legibility across 14 reasoning models, finding that RL often causes reasoning to become illegible to both humans and AI monitors, with reasoning models (except Claude) generating illegible CoTs while returning to perfectly readable final answers. We show that models use illegible reasoning to reach correct answers (accuracy dropping by 53\\% when forced to use only legible portions), yet find no correlation between legibility and performance when resampling - suggesting the relationship is more nuanced. We also find that legibility degrades on harder questions. We discuss potential hypotheses for these results, including steganography, training artifacts, and vestigial tokens. These results suggest that without explicit optimization for legibility, outcome-based RL naturally produces models with increasingly opaque reasoning processes, potentially undermining monitoring approaches.",
      "arxiv_id": "2510.27338v1",
      "published": "2025-10-31",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.27338v1",
      "pdf_url": "https://arxiv.org/pdf/2510.27338v1.pdf"
    },
    {
      "title": "SuperRL: Reinforcement Learning with Supervision to Boost Language Model Reasoning",
      "authors": [
        "Yihao Liu",
        "Shuocheng Li",
        "Lang Cao",
        "Yuhang Xie",
        "Mengyu Zhou",
        "Haoyu Dong",
        "Xiaojun Ma",
        "Shi Han",
        "Dongmei Zhang"
      ],
      "summary": "Large language models are increasingly used for complex reasoning tasks where high-quality offline data such as expert-annotated solutions and distilled reasoning traces are often available. However, in environments with sparse rewards, reinforcement learning struggles to sample successful trajectories, leading to inefficient learning. At the same time, these offline trajectories that represent correct reasoning paths are not utilized by standard on-policy reinforcement learning methods. We introduce SuperRL, a unified training framework that adaptively alternates between RL and SFT. Whenever every rollout for a given instance receives zero reward, indicating the absence of a learning signal, SuperRL falls back to SFT on the curated offline data. Extensive experiments across diverse reasoning benchmarks show that SuperRL surpasses vanilla RL by delivering higher sample efficiency, stronger generalization, and improved robustness under sparse rewards.",
      "arxiv_id": "2506.01096v2",
      "published": "2025-06-01",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.01096v2",
      "pdf_url": "https://arxiv.org/pdf/2506.01096v2.pdf"
    },
    {
      "title": "Can Large Language Models Develop Strategic Reasoning? Post-training Insights from Learning Chess",
      "authors": [
        "Dongyoon Hwang",
        "Hojoon Lee",
        "Jaegul Choo",
        "Dongmin Park",
        "Jongho Park"
      ],
      "summary": "While reinforcement learning (RL) for large language models (LLMs) has shown promise in mathematical reasoning, strategic reasoning for LLMs using RL remains largely unexplored. We investigate whether LLMs can develop strategic reasoning capabilities through RL in chess. To this end, we leverage a chess-pretrained action-value network to provide dense reward on the LLM's output move quality, which can be seen as a form of knowledge distillation. Our experiments show that our distillation-based dense rewards often outperform sparse binary rewards. However, surprisingly, all models plateau far below expert levels. We provide SFT and RL ablations on chess reasoning training and find evidence that this limitation stems from a deficit in the pretrained models' internal understanding of chess-a deficit which RL alone may not be able to fully overcome. The code is available at https://github.com/krafton-ai/Chess-R1.",
      "arxiv_id": "2507.00726v3",
      "published": "2025-07-01",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2507.00726v3",
      "pdf_url": "https://arxiv.org/pdf/2507.00726v3.pdf"
    },
    {
      "title": "Satori: Reinforcement Learning with Chain-of-Action-Thought Enhances LLM Reasoning via Autoregressive Search",
      "authors": [
        "Maohao Shen",
        "Guangtao Zeng",
        "Zhenting Qi",
        "Zhang-Wei Hong",
        "Zhenfang Chen",
        "Wei Lu",
        "Gregory Wornell",
        "Subhro Das",
        "David Cox",
        "Chuang Gan"
      ],
      "summary": "Large language models (LLMs) have demonstrated remarkable reasoning capabilities across diverse domains. Recent studies have shown that increasing test-time computation enhances LLMs' reasoning capabilities. This typically involves extensive sampling at inference time guided by an external LLM verifier, resulting in a two-player system. Despite external guidance, the effectiveness of this system demonstrates the potential of a single LLM to tackle complex tasks. Thus, we pose a new research problem: Can we internalize the searching capabilities to fundamentally enhance the reasoning abilities of a single LLM? This work explores an orthogonal direction focusing on post-training LLMs for autoregressive searching (i.e., an extended reasoning process with self-reflection and self-exploration of new strategies). To achieve this, we propose the Chain-of-Action-Thought (COAT) reasoning and a two-stage training paradigm: 1) a small-scale format tuning stage to internalize the COAT reasoning format and 2) a large-scale self-improvement stage leveraging reinforcement learning. Our approach results in Satori, a 7B LLM trained on open-source models and data. Extensive empirical evaluations demonstrate that Satori achieves state-of-the-art performance on mathematical reasoning benchmarks while exhibits strong generalization to out-of-domain tasks. Code, data, and models are fully open-sourced.",
      "arxiv_id": "2502.02508v3",
      "published": "2025-02-04",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2502.02508v3",
      "pdf_url": "https://arxiv.org/pdf/2502.02508v3.pdf"
    },
    {
      "title": "Reinforcement learning fine-tuning of language model for instruction following and math reasoning",
      "authors": [
        "Yifu Han",
        "Geo Zhang"
      ],
      "summary": "This study investigates the effectiveness of reinforcement learning (RL) fine-tuning techniques on a compact language model (Qwen2.5-0.5B Base) for two challenging tasks: instruction following and mathematical reasoning. We compare supervised fine-tuning (SFT), Direct Preference Optimization (DPO) using preference-labeled data, and Reinforce Leave-One-Out (RLOO) with reward models. Our experiments show that RLOO with DeBERTa reward modeling achieves the best alignment, while DPO provides strong and consistent results. For math reasoing tasks, synthetic data augmentation and best-of-N sampling with an external verifier significantly improve accuracy, showing the potential of combining fine-tuning with inference-time tools. This study highlights key trade-offs and practical strategies for training lightweight, task-aligned small-scale language models.",
      "arxiv_id": "2506.21560v2",
      "published": "2025-06-11",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.21560v2",
      "pdf_url": "https://arxiv.org/pdf/2506.21560v2.pdf"
    },
    {
      "title": "Perception, Reason, Think, and Plan: A Survey on Large Multimodal Reasoning Models",
      "authors": [
        "Yunxin Li",
        "Zhenyu Liu",
        "Zitao Li",
        "Xuanyu Zhang",
        "Zhenran Xu",
        "Xinyu Chen",
        "Haoyuan Shi",
        "Shenyuan Jiang",
        "Xintong Wang",
        "Jifang Wang",
        "Shouzheng Huang",
        "Xinping Zhao",
        "Borui Jiang",
        "Lanqing Hong",
        "Longyue Wang",
        "Zhuotao Tian",
        "Baoxing Huai",
        "Wenhan Luo",
        "Weihua Luo",
        "Zheng Zhang",
        "Baotian Hu",
        "Min Zhang"
      ],
      "summary": "Reasoning lies at the heart of intelligence, shaping the ability to make decisions, draw conclusions, and generalize across domains. In artificial intelligence, as systems increasingly operate in open, uncertain, and multimodal environments, reasoning becomes essential for enabling robust and adaptive behavior. Large Multimodal Reasoning Models (LMRMs) have emerged as a promising paradigm, integrating modalities such as text, images, audio, and video to support complex reasoning capabilities and aiming to achieve comprehensive perception, precise understanding, and deep reasoning. As research advances, multimodal reasoning has rapidly evolved from modular, perception-driven pipelines to unified, language-centric frameworks that offer more coherent cross-modal understanding. While instruction tuning and reinforcement learning have improved model reasoning, significant challenges remain in omni-modal generalization, reasoning depth, and agentic behavior. To address these issues, we present a comprehensive and structured survey of multimodal reasoning research, organized around a four-stage developmental roadmap that reflects the field's shifting design philosophies and emerging capabilities. First, we review early efforts based on task-specific modules, where reasoning was implicitly embedded across stages of representation, alignment, and fusion. Next, we examine recent approaches that unify reasoning into multimodal LLMs, with advances such as Multimodal Chain-of-Thought (MCoT) and multimodal reinforcement learning enabling richer and more structured reasoning chains. Finally, drawing on empirical insights from challenging benchmarks and experimental cases of OpenAI O3 and O4-mini, we discuss the conceptual direction of native large multimodal reasoning models (N-LMRMs), which aim to support scalable, agentic, and adaptive reasoning and planning in complex, real-world environments.",
      "arxiv_id": "2505.04921v2",
      "published": "2025-05-08",
      "categories": [
        "cs.CV",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.04921v2",
      "pdf_url": "https://arxiv.org/pdf/2505.04921v2.pdf"
    },
    {
      "title": "Understanding Reasoning in Chain-of-Thought from the Hopfieldian View",
      "authors": [
        "Lijie Hu",
        "Liang Liu",
        "Shu Yang",
        "Xin Chen",
        "Zhen Tan",
        "Muhammad Asif Ali",
        "Mengdi Li",
        "Di Wang"
      ],
      "summary": "Large Language Models have demonstrated remarkable abilities across various tasks, with Chain-of-Thought (CoT) prompting emerging as a key technique to enhance reasoning capabilities. However, existing research primarily focuses on improving performance, lacking a comprehensive framework to explain and understand the fundamental factors behind CoT's success. To bridge this gap, we introduce a novel perspective grounded in the Hopfieldian view of cognition in cognitive neuroscience. We establish a connection between CoT reasoning and key cognitive elements such as stimuli, actions, neural populations, and representation spaces. From our view, we can understand the reasoning process as the movement between these representation spaces. Building on this insight, we develop a method for localizing reasoning errors in the response of CoTs. Moreover, we propose the Representation-of-Thought (RoT) framework, which leverages the robustness of low-dimensional representation spaces to enhance the robustness of the reasoning process in CoTs. Experimental results demonstrate that RoT improves the robustness and interpretability of CoT reasoning while offering fine-grained control over the reasoning process.",
      "arxiv_id": "2410.03595v1",
      "published": "2024-10-04",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2410.03595v1",
      "pdf_url": "https://arxiv.org/pdf/2410.03595v1.pdf"
    },
    {
      "title": "CTRLS: Chain-of-Thought Reasoning via Latent State-Transition",
      "authors": [
        "Junda Wu",
        "Yuxin Xiong",
        "Xintong Li",
        "Zhengmian Hu",
        "Tong Yu",
        "Rui Wang",
        "Xiang Chen",
        "Jingbo Shang",
        "Julian McAuley"
      ],
      "summary": "Chain-of-thought (CoT) reasoning enables large language models (LLMs) to break down complex problems into interpretable intermediate steps, significantly enhancing model transparency and performance in reasoning tasks. However, conventional CoT methods rely on heuristic sampling without structured modeling of reasoning transitions, constraining their ability to systematically explore and discover diverse and effective reasoning trajectories. In this work, we introduce CTRLS, a framework that formulates CoT reasoning as a Markov decision process (MDP) with latent state transitions, enabling principled and state-aware exploration via distributional reinforcement learning. By modelling reasoning actions as explicit probability distributions in latent space, our approach explicitly models epistemic uncertainty, facilitating robust exploration of the reasoning space. As part of our framework, we introduce an on-policy reinforcement learning strategy incorporating epsilon-greedy exploration and entropy-based regularization to iteratively refine latent state transitions without requiring additional fine-tuning of the underlying LLM. Theoretical analyses provide evidence lower bounds (ELBO), theoretically grounding our transition-aware modeling of latent reasoning dynamics. Further experiments demonstrate improvements in reasoning accuracy, diversity, and exploration efficiency across benchmark reasoning tasks.",
      "arxiv_id": "2507.08182v1",
      "published": "2025-07-10",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2507.08182v1",
      "pdf_url": "https://arxiv.org/pdf/2507.08182v1.pdf"
    },
    {
      "title": "Perception Before Reasoning: Two-Stage Reinforcement Learning for Visual Reasoning in Vision-Language Models",
      "authors": [
        "Yan Chen",
        "Long Li",
        "Teng Xi",
        "Long Zeng",
        "Jingdong Wang"
      ],
      "summary": "Reinforcement learning (RL) has proven highly effective in eliciting the reasoning capabilities of large language models (LLMs). Inspired by this success, recent studies have explored applying similar techniques to vision-language models (VLMs), aiming to enhance their reasoning performance. However, directly transplanting RL methods from LLMs to VLMs is suboptimal, as the tasks faced by VLMs are inherently more complex. Specifically, VLMs must first accurately perceive and understand visual inputs before reasoning can be effectively performed. To address this challenge, we propose a two-stage reinforcement learning framework designed to jointly enhance both the perceptual and reasoning capabilities of VLMs. To mitigate the vanishing advantage issue commonly observed in RL training, we first perform dataset-level sampling to selectively strengthen specific capabilities using distinct data sources. During training, the first stage focuses on improving the model's visual perception through coarse- and fine-grained visual understanding, while the second stage targets the enhancement of reasoning abilities. After the proposed two-stage reinforcement learning process, we obtain PeBR-R1, a vision-language model with significantly enhanced perceptual and reasoning capabilities. Experimental results on seven benchmark datasets demonstrate the effectiveness of our approach and validate the superior performance of PeBR-R1 across diverse visual reasoning tasks.",
      "arxiv_id": "2509.13031v2",
      "published": "2025-09-16",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2509.13031v2",
      "pdf_url": "https://arxiv.org/pdf/2509.13031v2.pdf"
    },
    {
      "title": "AR$^2$: Adversarial Reinforcement Learning for Abstract Reasoning in Large Language Models",
      "authors": [
        "Cheng-Kai Yeh",
        "Hsing-Wang Lee",
        "Chung-Hung Kuo",
        "Hen-Hsen Huang"
      ],
      "summary": "Abstraction--the ability to recognize and distill essential computational patterns from complex problem statements--is a foundational skill in computer science, critical both for human problem-solvers and coding-oriented large language models (LLMs). Despite recent advances in training LLMs for code generation using reinforcement learning (RL), most existing approaches focus primarily on superficial pattern recognition, overlooking explicit training for abstraction. In this study, we propose AR$^2$ (Adversarial Reinforcement Learning for Abstract Reasoning), a novel framework explicitly designed to enhance the abstraction abilities of LLMs. AR$^2$ employs a teacher model to transform kernel problems into narrative-rich, challenging descriptions without changing their fundamental logic. Simultaneously, a student coding model is trained to solve these complex narrative problems by extracting their underlying computational kernels. Experimental results demonstrate that AR$^2$ substantially improves the student model's accuracy on previously unseen, challenging programming tasks, underscoring abstraction as a key skill for enhancing LLM generalization.",
      "arxiv_id": "2509.03537v1",
      "published": "2025-08-27",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2509.03537v1",
      "pdf_url": "https://arxiv.org/pdf/2509.03537v1.pdf"
    },
    {
      "title": "Fin-R1: A Large Language Model for Financial Reasoning through Reinforcement Learning",
      "authors": [
        "Zhaowei Liu",
        "Xin Guo",
        "Fangqi Lou",
        "Lingfeng Zeng",
        "Jinyi Niu",
        "Zixuan Wang",
        "Jiajie Xu",
        "Weige Cai",
        "Ziwei Yang",
        "Xueqian Zhao",
        "Chao Li",
        "Sheng Xu",
        "Dezhi Chen",
        "Yun Chen",
        "Zuo Bai",
        "Liwen Zhang"
      ],
      "summary": "Reasoning large language models are rapidly evolving across various domains. However, their capabilities in handling complex financial tasks still require in-depth exploration. In this paper, we introduce Fin-R1, a reasoning large language model specifically designed for the financial sector. Fin-R1 is built using a two-stage architecture, leveraging a financial reasoning dataset distilled and processed based on DeepSeek-R1. Through supervised fine-tuning (SFT) and reinforcement learning (RL) training, it demonstrates performance close to DeepSeek-R1 with a parameter size of 7 billion across a range of financial reasoning tasks. It achieves the state-of-the-art (SOTA) in the FinQA and ConvFinQA tasks between those LLMs in our evaluation, surpassing larger models in other tasks as well. Fin-R1 showcases strong reasoning and decision-making capabilities, providing solutions to various problems encountered in the financial domain. Our code is available at https://github.com/SUFE-AIFLM-Lab/Fin-R1.",
      "arxiv_id": "2503.16252v2",
      "published": "2025-03-20",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2503.16252v2",
      "pdf_url": "https://arxiv.org/pdf/2503.16252v2.pdf"
    },
    {
      "title": "Training Large Language Models for Reasoning through Reverse Curriculum Reinforcement Learning",
      "authors": [
        "Zhiheng Xi",
        "Wenxiang Chen",
        "Boyang Hong",
        "Senjie Jin",
        "Rui Zheng",
        "Wei He",
        "Yiwen Ding",
        "Shichun Liu",
        "Xin Guo",
        "Junzhe Wang",
        "Honglin Guo",
        "Wei Shen",
        "Xiaoran Fan",
        "Yuhao Zhou",
        "Shihan Dou",
        "Xiao Wang",
        "Xinbo Zhang",
        "Peng Sun",
        "Tao Gui",
        "Qi Zhang",
        "Xuanjing Huang"
      ],
      "summary": "In this paper, we propose R$^3$: Learning Reasoning through Reverse Curriculum Reinforcement Learning (RL), a novel method that employs only outcome supervision to achieve the benefits of process supervision for large language models. The core challenge in applying RL to complex reasoning is to identify a sequence of actions that result in positive rewards and provide appropriate supervision for optimization. Outcome supervision provides sparse rewards for final results without identifying error locations, whereas process supervision offers step-wise rewards but requires extensive manual annotation. R$^3$ overcomes these limitations by learning from correct demonstrations. Specifically, R$^3$ progressively slides the start state of reasoning from a demonstration's end to its beginning, facilitating easier model exploration at all stages. Thus, R$^3$ establishes a step-wise curriculum, allowing outcome supervision to offer step-level signals and precisely pinpoint errors. Using Llama2-7B, our method surpasses RL baseline on eight reasoning tasks by $4.1$ points on average. Notebaly, in program-based reasoning on GSM8K, it exceeds the baseline by $4.2$ points across three backbone models, and without any extra data, Codellama-7B + R$^3$ performs comparable to larger models or closed-source models.",
      "arxiv_id": "2402.05808v2",
      "published": "2024-02-08",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2402.05808v2",
      "pdf_url": "https://arxiv.org/pdf/2402.05808v2.pdf"
    },
    {
      "title": "Integrating Large Language Models and Reinforcement Learning for Non-Linear Reasoning",
      "authors": [
        "Yoav Alon",
        "Cristina David"
      ],
      "summary": "Large Language Models (LLMs) were shown to struggle with long-term planning, which may be caused by the limited way in which they explore the space of possible solutions. We propose an architecture where a Reinforcement Learning (RL) Agent guides an LLM's space exploration: (1) the Agent has access to domain-specific information, and can therefore make decisions about the quality of candidate solutions based on specific and relevant metrics, which were not explicitly considered by the LLM's training objective; (2) the LLM can focus on generating immediate next steps, without the need for long-term planning. We allow non-linear reasoning by exploring alternative paths and backtracking. We evaluate this architecture on the program equivalence task, and compare it against Chain of Thought (CoT) and Tree of Thoughts (ToT). We assess both the downstream task, denoting the binary classification, and the intermediate reasoning steps. Our approach compares positively against CoT and ToT.",
      "arxiv_id": "2410.13501v1",
      "published": "2024-10-17",
      "categories": [
        "cs.LG",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2410.13501v1",
      "pdf_url": "https://arxiv.org/pdf/2410.13501v1.pdf"
    },
    {
      "title": "On the Impact of Fine-Tuning on Chain-of-Thought Reasoning",
      "authors": [
        "Elita Lobo",
        "Chirag Agarwal",
        "Himabindu Lakkaraju"
      ],
      "summary": "Large language models have emerged as powerful tools for general intelligence, showcasing advanced natural language processing capabilities that find applications across diverse domains. Despite their impressive performance, recent studies have highlighted the potential for significant enhancements in LLMs' task-specific performance through fine-tuning strategies like Reinforcement Learning with Human Feedback (RLHF), supervised fine-tuning (SFT), and Quantized Low-Rank Adapters (Q-LoRA) method. However, previous works have shown that while fine-tuning offers significant performance gains, it also leads to challenges such as catastrophic forgetting and privacy and safety risks. To this end, there has been little to no work in \\textit{understanding the impact of fine-tuning on the reasoning capabilities of LLMs}. Our research investigates the effect of fine-tuning on the reasoning abilities of LLMs, addressing critical questions regarding the impact of task-specific fine-tuning on overall reasoning capabilities, the influence of fine-tuning on Chain-of-Thought (CoT) reasoning performance, and the implications for the faithfulness of CoT reasonings. By exploring these dimensions, our study shows the impact of fine-tuning on LLM reasoning capabilities, where the faithfulness of CoT reasoning, on average across four datasets, decreases, highlighting potential shifts in internal mechanisms of the LLMs resulting from fine-tuning processes.",
      "arxiv_id": "2411.15382v2",
      "published": "2024-11-22",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2411.15382v2",
      "pdf_url": "https://arxiv.org/pdf/2411.15382v2.pdf"
    },
    {
      "title": "Advancing Multimodal Reasoning via Reinforcement Learning with Cold Start",
      "authors": [
        "Lai Wei",
        "Yuting Li",
        "Kaipeng Zheng",
        "Chen Wang",
        "Yue Wang",
        "Linghe Kong",
        "Lichao Sun",
        "Weiran Huang"
      ],
      "summary": "Recent advancements in large language models (LLMs) have demonstrated impressive chain-of-thought reasoning capabilities, with reinforcement learning (RL) playing a crucial role in this progress. While \"aha moment\" patterns--where models exhibit self-correction through reflection--are often attributed to emergent properties from RL, we first demonstrate that these patterns exist in multimodal LLMs (MLLMs) prior to RL training but may not necessarily correlate with improved reasoning performance. Building on these insights, we present a comprehensive study on enhancing multimodal reasoning through a two-stage approach: (1) supervised fine-tuning (SFT) as a cold start with structured chain-of-thought reasoning patterns, followed by (2) reinforcement learning via GRPO to further refine these capabilities. Our extensive experiments show that this combined approach consistently outperforms both SFT-only and RL-only methods across challenging multimodal reasoning benchmarks. The resulting models achieve state-of-the-art performance among open-source MLLMs at both 3B and 7B scales, with our 7B model showing substantial improvements over base models (e.g., 66.3 %$\\rightarrow$73.4 % on MathVista, 62.9 %$\\rightarrow$70.4 % on We-Math) and our 3B model achieving performance competitive with several 7B models. Overall, this work provides practical guidance for building advanced multimodal reasoning models. Our code is available at https://github.com/waltonfuture/RL-with-Cold-Start.",
      "arxiv_id": "2505.22334v2",
      "published": "2025-05-28",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.CV",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.22334v2",
      "pdf_url": "https://arxiv.org/pdf/2505.22334v2.pdf"
    },
    {
      "title": "Zebra-CoT: A Dataset for Interleaved Vision Language Reasoning",
      "authors": [
        "Ang Li",
        "Charles Wang",
        "Deqing Fu",
        "Kaiyu Yue",
        "Zikui Cai",
        "Wang Bill Zhu",
        "Ollie Liu",
        "Peng Guo",
        "Willie Neiswanger",
        "Furong Huang",
        "Tom Goldstein",
        "Micah Goldblum"
      ],
      "summary": "Humans often use visual aids, for example diagrams or sketches, when solving complex problems. Training multimodal models to do the same, known as Visual Chain of Thought (Visual CoT), is challenging due to: (1) poor off-the-shelf visual CoT performance, which hinders reinforcement learning, and (2) the lack of high-quality visual CoT training data. We introduce $\\textbf{Zebra-CoT}$, a diverse large-scale dataset with 182,384 samples, containing logically coherent interleaved text-image reasoning traces. We focus on four categories of tasks where sketching or visual reasoning is especially natural, spanning scientific questions such as geometry, physics, and algorithms; 2D visual reasoning tasks like visual search and jigsaw puzzles; 3D reasoning tasks including 3D multi-hop inference, embodied and robot planning; visual logic problems and strategic games like chess. Fine-tuning the Anole-7B model on the Zebra-CoT training corpus results in an improvement of +12% in our test-set accuracy and yields up to +13% performance gain on standard VLM benchmark evaluations. Fine-tuning Bagel-7B yields a model that generates high-quality interleaved visual reasoning chains, underscoring Zebra-CoT's effectiveness for developing multimodal reasoning abilities. We open-source our dataset and models to support development and evaluation of visual CoT.",
      "arxiv_id": "2507.16746v2",
      "published": "2025-07-22",
      "categories": [
        "cs.CV",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2507.16746v2",
      "pdf_url": "https://arxiv.org/pdf/2507.16746v2.pdf"
    },
    {
      "title": "Route-and-Reason: Scaling Large Language Model Reasoning with Reinforced Model Router",
      "authors": [
        "Chenyang Shao",
        "Xinyang Liu",
        "Yutang Lin",
        "Fengli Xu",
        "Yong Li"
      ],
      "summary": "Multi-step reasoning has proven essential for enhancing the problem-solving capabilities of Large Language Models (LLMs) by decomposing complex tasks into intermediate steps, either explicitly or implicitly. Extending the reasoning chain at test time through deeper thought processes or broader exploration, can furthur improve performance, but often incurs substantial costs due to the explosion in token usage. Yet, many reasoning steps are relatively simple and can be handled by more efficient smaller-scale language models (SLMs). This motivates hybrid approaches that allocate subtasks across models of varying capacities. However, realizing such collaboration requires accurate task decomposition and difficulty-aware subtask allocation, which is challenging. To address this, we propose R2-Reasoner, a novel framework that enables collaborative reasoning across heterogeneous LLMs by dynamically routing sub-tasks based on estimated complexity. At the core of our framework is a Reinforced Model Router, composed of a task decomposer and a subtask allocator. The task decomposer segments complex input queries into logically ordered subtasks, while the subtask allocator assigns each subtask to the most appropriate model, ranging from lightweight SLMs to powerful LLMs, balancing accuracy and efficiency. To train this router, we introduce a staged pipeline that combines supervised fine-tuning on task-specific datasets with Group Relative Policy Optimization algorithm, enabling self-supervised refinement through iterative reinforcement learning. Extensive experiments across four challenging benchmarks demonstrate that R2-Reasoner reduces API costs by 86.85% while maintaining or surpassing baseline accuracy. Our framework paves the way for more cost-effective and adaptive LLM reasoning. The code is open-source at https://anonymous.4open.science/r/R2_Reasoner .",
      "arxiv_id": "2506.05901v1",
      "published": "2025-06-06",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.05901v1",
      "pdf_url": "https://arxiv.org/pdf/2506.05901v1.pdf"
    },
    {
      "title": "Tagging the Thought: Unlocking Personalization Reasoning via Reinforcement Learning",
      "authors": [
        "Song Jin",
        "Juntian Zhang",
        "Yong Liu",
        "Xun Zhang",
        "Yufei Zhang",
        "Fei Jiang",
        "Guojun Yin",
        "Wei Lin",
        "Rui Yan"
      ],
      "summary": "Recent advancements have endowed Large Language Models (LLMs) with impressive general reasoning capabilities, yet they often struggle with personalization reasoning - the crucial ability to analyze user history, infer unique preferences, and generate tailored responses. To address this limitation, we introduce TagPR, a novel training framework that significantly enhances an LLM's intrinsic capacity for personalization reasoning through a tagging the thought approach. Our method first develops a data-driven pipeline to automatically generate and semantically label reasoning chains, creating a structured dataset that fosters interpretable reasoning. We then propose a synergistic training strategy that begins with Supervised Fine-Tuning (SFT) on this tagged data to establish foundational reasoning patterns, followed by a multi-stage reinforcement learning (RL) process. This RL phase is guided by a unique composite reward signal, which integrates tag-based constraints and a novel Personalization Reward Model with User Embeddings (PRMU) to achieve fine-grained alignment with user-specific logic. Extensive experiments on the public LaMP benchmark and a self-constructed dataset demonstrate that our approach achieves state-of-the-art results, delivering an average improvement of 32.65% over the base model across all tasks. Our work validates that structured, interpretable reasoning is a highly effective pathway to unlocking genuine personalization capabilities in LLMs.",
      "arxiv_id": "2509.23140v1",
      "published": "2025-09-27",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.23140v1",
      "pdf_url": "https://arxiv.org/pdf/2509.23140v1.pdf"
    },
    {
      "title": "RACE-Align: Retrieval-Augmented and Chain-of-Thought Enhanced Preference Alignment for Large Language Models",
      "authors": [
        "Qihang Yan",
        "Xinyu Zhang",
        "Luming Guo",
        "Qi Zhang",
        "Feifan Liu"
      ],
      "summary": "Large Language Models (LLMs) struggle with accuracy, domain-specific reasoning, and interpretability in vertical domains. Traditional preference alignment methods like Reinforcement Learning from Human Feedback (RLHF) and Direct Preference Optimization (DPO) often overlook the underlying knowledge sources and reasoning logic. This paper introduces RACE-Align (Retrieval-Augmented and Chain-of-Thought Enhanced Alignment), a novel framework designed to address these limitations. RACE-Align systematically constructs a binary preference dataset incorporating external knowledge support and explicit Chain-of-Thought (CoT) reasoning, then aligns LLMs using the DPO algorithm. The core innovation lies in its preference data construction strategy: it integrates AI-driven retrieval for factual grounding, enhancing knowledgeability and accuracy, and emphasizes the optimization of domain-specific CoT, treating the reasoning process itself as a key preference dimension. A multi-stage, AI-driven refinement pipeline cost-effectively generates these preference pairs. Experimental validation in Traditional Chinese Medicine (TCM) using Qwen3-1.7B as the base model demonstrates that RACE-Align significantly outperforms the original base model and a model fine-tuned only with Supervised Fine-Tuning (SFT). Improvements were observed across multiple dimensions, including answer accuracy, information richness, application of TCM thinking patterns, logicality and depth of reasoning, and interpretability. These findings suggest RACE-Align offers an effective pathway to enhance LLMs' knowledge application, reasoning reliability, and process transparency in complex vertical domains.",
      "arxiv_id": "2506.02726v1",
      "published": "2025-06-03",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2506.02726v1",
      "pdf_url": "https://arxiv.org/pdf/2506.02726v1.pdf"
    },
    {
      "title": "Explore Data Left Behind in Reinforcement Learning for Reasoning Language Models",
      "authors": [
        "Chenxi Liu",
        "Junjie Liang",
        "Yuqi Jia",
        "Bochuan Cao",
        "Yang Bai",
        "Heng Huang",
        "Xun Chen"
      ],
      "summary": "Reinforcement Learning with Verifiable Rewards (RLVR) has emerged as an effective approach for improving the reasoning abilities of large language models (LLMs). The Group Relative Policy Optimization (GRPO) family has demonstrated strong performance in training LLMs with RLVR. However, as models train longer and scale larger, more training prompts become residual prompts, those with zero variance rewards that provide no training signal. Consequently, fewer prompts contribute to training, reducing diversity and hindering effectiveness. To fully exploit these residual prompts, we propose the Explore Residual Prompts in Policy Optimization (ERPO) framework, which encourages exploration on residual prompts and reactivates their training signals. ERPO maintains a history tracker for each prompt and adaptively increases the sampling temperature for residual prompts that previously produced all correct responses. This encourages the model to generate more diverse reasoning traces, introducing incorrect responses that revive training signals. Empirical results on the Qwen2.5 series demonstrate that ERPO consistently surpasses strong baselines across multiple mathematical reasoning benchmarks.",
      "arxiv_id": "2511.04800v1",
      "published": "2025-11-06",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2511.04800v1",
      "pdf_url": "https://arxiv.org/pdf/2511.04800v1.pdf"
    },
    {
      "title": "Making Small Language Models Efficient Reasoners: Intervention, Supervision, Reinforcement",
      "authors": [
        "Xuechen Zhang",
        "Zijian Huang",
        "Chenshun Ni",
        "Ziyang Xiong",
        "Jiasi Chen",
        "Samet Oymak"
      ],
      "summary": "Recent research enhances language model reasoning by scaling test-time compute via longer chain-of-thought traces. This often improves accuracy but also introduces redundancy and high computational cost, especially for small language models distilled with supervised fine-tuning (SFT). In this work, we propose new algorithms to improve token-efficient reasoning with small-scale models by effectively trading off accuracy and computation. We first show that the post-SFT model fails to determine the optimal stopping point of the reasoning process, resulting in verbose and repetitive outputs. Verbosity also significantly varies across wrong vs correct responses. To address these issues, we propose two solutions: (1) Temperature scaling (TS) to control the stopping point for the thinking phase and thereby trace length, and (2) TLDR: a length-regularized reinforcement learning method based on GRPO that facilitates multi-level trace length control (e.g. short, medium, long reasoning). Experiments on four reasoning benchmarks, MATH500, AMC, AIME24 and OlympiadBench, demonstrate that TS is highly effective compared to s1's budget forcing approach and TLDR significantly improves token efficiency by about 50% with minimal to no accuracy loss over the SFT baseline. Moreover, TLDR also facilitates flexible control over the response length, offering a practical and effective solution for token-efficient reasoning in small models. Ultimately, our work reveals the importance of stopping time control, highlights shortcomings of pure SFT, and provides effective algorithmic recipes.",
      "arxiv_id": "2505.07961v3",
      "published": "2025-05-12",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.07961v3",
      "pdf_url": "https://arxiv.org/pdf/2505.07961v3.pdf"
    },
    {
      "title": "Fleming-R1: Toward Expert-Level Medical Reasoning via Reinforcement Learning",
      "authors": [
        "Chi Liu",
        "Derek Li",
        "Yan Shu",
        "Robin Chen",
        "Derek Duan",
        "Teng Fang",
        "Bryan Dai"
      ],
      "summary": "While large language models show promise in medical applications, achieving expert-level clinical reasoning remains challenging due to the need for both accurate answers and transparent reasoning processes. To address this challenge, we introduce Fleming-R1, a model designed for verifiable medical reasoning through three complementary innovations. First, our Reasoning-Oriented Data Strategy (RODS) combines curated medical QA datasets with knowledge-graph-guided synthesis to improve coverage of underrepresented diseases, drugs, and multi-hop reasoning chains. Second, we employ Chain-of-Thought (CoT) cold start to distill high-quality reasoning trajectories from teacher models, establishing robust inference priors. Third, we implement a two-stage Reinforcement Learning from Verifiable Rewards (RLVR) framework using Group Relative Policy Optimization, which consolidates core reasoning skills while targeting persistent failure modes through adaptive hard-sample mining. Across diverse medical benchmarks, Fleming-R1 delivers substantial parameter-efficient improvements: the 7B variant surpasses much larger baselines, while the 32B model achieves near-parity with GPT-4o and consistently outperforms strong open-source alternatives. These results demonstrate that structured data design, reasoning-oriented initialization, and verifiable reinforcement learning can advance clinical reasoning beyond simple accuracy optimization. We release Fleming-R1 publicly to promote transparent, reproducible, and auditable progress in medical AI, enabling safer deployment in high-stakes clinical environments.",
      "arxiv_id": "2509.15279v1",
      "published": "2025-09-18",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.15279v1",
      "pdf_url": "https://arxiv.org/pdf/2509.15279v1.pdf"
    },
    {
      "title": "Exploring Advanced Large Language Models with LLMsuite",
      "authors": [
        "Giorgio Roffo"
      ],
      "summary": "This tutorial explores the advancements and challenges in the development of Large Language Models (LLMs) such as ChatGPT and Gemini. It addresses inherent limitations like temporal knowledge cutoffs, mathematical inaccuracies, and the generation of incorrect information, proposing solutions like Retrieval Augmented Generation (RAG), Program-Aided Language Models (PAL), and frameworks such as ReAct and LangChain. The integration of these techniques enhances LLM performance and reliability, especially in multi-step reasoning and complex task execution. The paper also covers fine-tuning strategies, including instruction fine-tuning, parameter-efficient methods like LoRA, and Reinforcement Learning from Human Feedback (RLHF) as well as Reinforced Self-Training (ReST). Additionally, it provides a comprehensive survey of transformer architectures and training techniques for LLMs. The source code can be accessed by contacting the author via email for a request.",
      "arxiv_id": "2407.12036v2",
      "published": "2024-07-01",
      "categories": [
        "cs.CL",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2407.12036v2",
      "pdf_url": "https://arxiv.org/pdf/2407.12036v2.pdf"
    },
    {
      "title": "The Reasoning Boundary Paradox: How Reinforcement Learning Constrains Language Models",
      "authors": [
        "Phuc Minh Nguyen",
        "Chinh D. La",
        "Duy M. H. Nguyen",
        "Nitesh V. Chawla",
        "Binh T. Nguyen",
        "Khoa D. Doan"
      ],
      "summary": "Reinforcement Learning with Verifiable Rewards (RLVR) has emerged as a key method for improving Large Language Models' reasoning capabilities, yet recent evidence suggests it may paradoxically shrink the reasoning boundary rather than expand it. This paper investigates the shrinkage issue of RLVR by analyzing its learning dynamics and reveals two critical phenomena that explain this failure. First, we expose negative interference in RLVR, where learning to solve certain training problems actively reduces the likelihood of correct solutions for others, leading to the decline of Pass@$k$ performance, or the probability of generating a correct solution within $k$ attempts. Second, we uncover the winner-take-all phenomenon: RLVR disproportionately reinforces problems with high likelihood, correct solutions, under the base model, while suppressing other initially low-likelihood ones. Through extensive theoretical and empirical analysis on multiple mathematical reasoning benchmarks, we show that this effect arises from the inherent on-policy sampling in standard RL objectives, causing the model to converge toward narrow solution strategies. Based on these insights, we propose a simple yet effective data curation algorithm that focuses RLVR learning on low-likelihood problems, achieving notable improvement in Pass@$k$ performance. Our code is available at https://github.com/mail-research/SELF-llm-interference.",
      "arxiv_id": "2510.02230v1",
      "published": "2025-10-02",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2510.02230v1",
      "pdf_url": "https://arxiv.org/pdf/2510.02230v1.pdf"
    },
    {
      "title": "S-GRPO: Early Exit via Reinforcement Learning in Reasoning Models",
      "authors": [
        "Muzhi Dai",
        "Chenxu Yang",
        "Qingyi Si"
      ],
      "summary": "As Test-Time Scaling emerges as an active research focus in the large language model community, advanced post-training methods increasingly emphasize extending chain-of-thought (CoT) generation length, thereby enhancing reasoning capabilities to approach Deepseek R1-like reasoning models. However, recent studies reveal that reasoning models (even Qwen3) consistently exhibit excessive thought redundancy in CoT generation. This overthinking issue arises from the inherent limitations of conventional outcome-reward reinforcement learning, which systematically overlooks the regulation of intermediate reasoning processes. This paper introduces Serial-Group Decaying-Reward Policy Optimization (S-GRPO), a novel reinforcement learning paradigm that enables models to implicitly evaluate the sufficiency of intermediate reasoning steps, thereby facilitating early exit in CoT generation. Unlike GRPO, which samples multiple possible reasoning paths in parallel (parallel group), S-GRPO only samples one reasoning path and serially selects multiple temporal positions from the path to exit thinking and directly generate answers (serial group). For correct answers within a serial group, rewards gradually decrease based on the exit positions along the reasoning path from front to back. This design encourages the model to produce more accurate and concise thoughts, while also incentivizing early thinking termination when appropriate. Empirical evaluations demonstrate that S-GRPO is compatible with state-of-the-art reasoning models, including Qwen3 and Deepseek-distill. Across diverse benchmarks such as GSM8K, AIME 2024, AMC 2023, MATH-500, and GPQA Diamond, S-GRPO achieves a substantial reduction in sequence length (35.4% - 61.1%) while simultaneously improving accuracy (absolute 0.72% - 6.08%).",
      "arxiv_id": "2505.07686v2",
      "published": "2025-05-12",
      "categories": [
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.07686v2",
      "pdf_url": "https://arxiv.org/pdf/2505.07686v2.pdf"
    },
    {
      "title": "Vision-Language Models are Zero-Shot Reward Models for Reinforcement Learning",
      "authors": [
        "Juan Rocamonde",
        "Victoriano Montesinos",
        "Elvis Nava",
        "Ethan Perez",
        "David Lindner"
      ],
      "summary": "Reinforcement learning (RL) requires either manually specifying a reward function, which is often infeasible, or learning a reward model from a large amount of human feedback, which is often very expensive. We study a more sample-efficient alternative: using pretrained vision-language models (VLMs) as zero-shot reward models (RMs) to specify tasks via natural language. We propose a natural and general approach to using VLMs as reward models, which we call VLM-RMs. We use VLM-RMs based on CLIP to train a MuJoCo humanoid to learn complex tasks without a manually specified reward function, such as kneeling, doing the splits, and sitting in a lotus position. For each of these tasks, we only provide a single sentence text prompt describing the desired task with minimal prompt engineering. We provide videos of the trained agents at: https://sites.google.com/view/vlm-rm. We can improve performance by providing a second \"baseline\" prompt and projecting out parts of the CLIP embedding space irrelevant to distinguish between goal and baseline. Further, we find a strong scaling effect for VLM-RMs: larger VLMs trained with more compute and data are better reward models. The failure modes of VLM-RMs we encountered are all related to known capability limitations of current VLMs, such as limited spatial reasoning ability or visually unrealistic environments that are far off-distribution for the VLM. We find that VLM-RMs are remarkably robust as long as the VLM is large enough. This suggests that future VLMs will become more and more useful reward models for a wide range of RL applications.",
      "arxiv_id": "2310.12921v2",
      "published": "2023-10-19",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2310.12921v2",
      "pdf_url": "https://arxiv.org/pdf/2310.12921v2.pdf"
    },
    {
      "title": "Language Model Self-improvement by Reinforcement Learning Contemplation",
      "authors": [
        "Jing-Cheng Pang",
        "Pengyuan Wang",
        "Kaiyuan Li",
        "Xiong-Hui Chen",
        "Jiacheng Xu",
        "Zongzhang Zhang",
        "Yang Yu"
      ],
      "summary": "Large Language Models (LLMs) have exhibited remarkable performance across various natural language processing (NLP) tasks. However, fine-tuning these models often necessitates substantial supervision, which can be expensive and time-consuming to obtain. This paper introduces a novel unsupervised method called LanguageModel Self-Improvement by Reinforcement Learning Contemplation (SIRLC) that improves LLMs without reliance on external labels. Our approach is grounded in the observation that it is simpler for language models to assess text quality than to generate text. Building on this insight, SIRLC assigns LLMs dual roles as both student and teacher. As a student, the LLM generates answers to unlabeled questions, while as a teacher, it evaluates the generated text and assigns scores accordingly. The model parameters are updated using reinforcement learning to maximize the evaluation score. We demonstrate that SIRLC can be applied to various NLP tasks, such as reasoning problems, text generation, and machine translation. Our experiments show that SIRLC effectively improves LLM performance without external supervision, resulting in a 5.6% increase in answering accuracy for reasoning tasks and a rise in BERTScore from 0.82 to 0.86 for translation tasks. Furthermore, SIRLC can be applied to models of different sizes, showcasing its broad applicability.",
      "arxiv_id": "2305.14483v1",
      "published": "2023-05-23",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2305.14483v1",
      "pdf_url": "https://arxiv.org/pdf/2305.14483v1.pdf"
    },
    {
      "title": "Exploring Chain-of-Thought Reasoning for Steerable Pluralistic Alignment",
      "authors": [
        "Yunfan Zhang",
        "Kathleen McKeown",
        "Smaranda Muresan"
      ],
      "summary": "Large Language Models (LLMs) are typically trained to reflect a relatively uniform set of values, which limits their applicability to tasks that require understanding of nuanced human perspectives. Recent research has underscored the importance of enabling LLMs to support steerable pluralism -- the capacity to adopt a specific perspective and align generated outputs with it. In this work, we investigate whether Chain-of-Thought (CoT) reasoning techniques can be applied to building steerable pluralistic models. We explore several methods, including CoT prompting, fine-tuning on human-authored CoT, fine-tuning on synthetic explanations, and Reinforcement Learning with Verifiable Rewards (RLVR). We evaluate these approaches using the Value Kaleidoscope and OpinionQA datasets. Among the methods studied, RLVR consistently outperforms others and demonstrates strong training sample efficiency. We further analyze the generated CoT traces with respect to faithfulness and safety.",
      "arxiv_id": "2510.04045v1",
      "published": "2025-10-05",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.04045v1",
      "pdf_url": "https://arxiv.org/pdf/2510.04045v1.pdf"
    },
    {
      "title": "NOVER: Incentive Training for Language Models via Verifier-Free Reinforcement Learning",
      "authors": [
        "Wei Liu",
        "Siya Qi",
        "Xinyu Wang",
        "Chen Qian",
        "Yali Du",
        "Yulan He"
      ],
      "summary": "Recent advances such as DeepSeek R1-Zero highlight the effectiveness of incentive training, a reinforcement learning paradigm that computes rewards solely based on the final answer part of a language model's output, thereby encouraging the generation of intermediate reasoning steps. However, these methods fundamentally rely on external verifiers, which limits their applicability to domains like mathematics and coding where such verifiers are readily available. Although reward models can serve as verifiers, they require high-quality annotated data and are costly to train. In this work, we propose NOVER, NO-VERifier Reinforcement Learning, a general reinforcement learning framework that requires only standard supervised fine-tuning data with no need for an external verifier. NOVER enables incentive training across a wide range of text-to-text tasks and outperforms the model of the same size distilled from large reasoning models such as DeepSeek R1 671B by 7.7 percent. Moreover, the flexibility of NOVER enables new possibilities for optimizing large language models, such as inverse incentive training.",
      "arxiv_id": "2505.16022v2",
      "published": "2025-05-21",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.16022v2",
      "pdf_url": "https://arxiv.org/pdf/2505.16022v2.pdf"
    },
    {
      "title": "Teaching Models to Verbalize Reward Hacking in Chain-of-Thought Reasoning",
      "authors": [
        "Miles Turpin",
        "Andy Arditi",
        "Marvin Li",
        "Joe Benton",
        "Julian Michael"
      ],
      "summary": "Language models trained with reinforcement learning (RL) can engage in reward hacking--the exploitation of unintended strategies for high reward--without revealing this behavior in their chain-of-thought reasoning. This makes the detection of reward hacking difficult, posing risks for high-stakes applications. We propose verbalization fine-tuning (VFT), a pre-RL fine-tuning intervention that trains models to explicitly acknowledge when they are influenced by prompt cues--hints which point to incorrect answers (e.g., \"a Stanford professor thinks the answer is A\"). To evaluate VFT, we subsequently train models with RL on environments where held-out prompt cues signal which incorrect answers will receive high reward, incentivizing models to exploit these cues instead of reasoning correctly. We measure how often models exploit these cues without verbalizing it. After RL, only 6% of the VFT-trained model's responses consist of undetected reward hacks. In comparison, when we perform RL without VFT, the rate of undetected reward hacks goes up to 88%; with a debiasing baseline intervention, this increases further to 99%. VFT achieves this by substantially increasing how often models verbalize the influence of cues, from 8% to 43% after VFT, and up to 94% after RL. Baselines remain low even after RL (11% and 1%). Our results show that teaching models to explicitly verbalize reward hacking behavior before RL significantly improves their detection, offering a practical path toward more transparent and safe AI systems.",
      "arxiv_id": "2506.22777v2",
      "published": "2025-06-28",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.22777v2",
      "pdf_url": "https://arxiv.org/pdf/2506.22777v2.pdf"
    },
    {
      "title": "UloRL:An Ultra-Long Output Reinforcement Learning Approach for Advancing Large Language Models' Reasoning Abilities",
      "authors": [
        "Dong Du",
        "Shulin Liu",
        "Tao Yang",
        "Shaohua Chen",
        "Yang Li"
      ],
      "summary": "Recent advances in large language models (LLMs) have highlighted the potential of reinforcement learning with verifiable rewards (RLVR) to enhance reasoning capabilities through extended output sequences. However, traditional RL frameworks face inefficiencies when handling ultra-long outputs due to long-tail sequence distributions and entropy collapse during training. To address these challenges, we propose an Ultra-Long Output Reinforcement Learning (UloRL) approach for advancing large language models' reasoning abilities. Specifically, we divide ultra long output decoding into short segments, enabling efficient training by mitigating delays caused by long-tail samples. Additionally, we introduce dynamic masking of well-Mastered Positive Tokens (MPTs) to prevent entropy collapse. Experimental results demonstrate the effectiveness of our approach. On the Qwen3-30B-A3B model, RL with segment rollout achieved 2.06x increase in training speed, while RL training with 128k-token outputs improves the model's performance on AIME2025 from 70.9\\% to 85.1\\% and on BeyondAIME from 50.7\\% to 61.9\\%, even surpassing Qwen3-235B-A22B with remarkable gains. These findings underscore the potential of our methods to advance the reasoning capabilities of LLMs with ultra-long sequence generation. We will release our code and model for further use by the community.",
      "arxiv_id": "2507.19766v1",
      "published": "2025-07-26",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2507.19766v1",
      "pdf_url": "https://arxiv.org/pdf/2507.19766v1.pdf"
    },
    {
      "title": "UniRL-Zero: Reinforcement Learning on Unified Models with Joint Language Model and Diffusion Model Experts",
      "authors": [
        "Fu-Yun Wang",
        "Han Zhang",
        "Michael Gharbi",
        "Hongsheng Li",
        "Taesung Park"
      ],
      "summary": "We present UniRL-Zero, a unified reinforcement learning (RL) framework that boosts, multimodal language model understanding and reasoning, diffusion model multimedia generation, and their beneficial interaction capabilities within a unified model. Our work defines six scenarios for unified model reinforcement learning, providing systematic baselines for reinforcement learning of unified understanding and generation model. Our code is available at https://github.com/G-U-N/UniRL.",
      "arxiv_id": "2510.17937v1",
      "published": "2025-10-20",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.17937v1",
      "pdf_url": "https://arxiv.org/pdf/2510.17937v1.pdf"
    },
    {
      "title": "Infi-MMR: Curriculum-based Unlocking Multimodal Reasoning via Phased Reinforcement Learning in Multimodal Small Language Models",
      "authors": [
        "Zeyu Liu",
        "Yuhang Liu",
        "Guanghao Zhu",
        "Congkai Xie",
        "Zhen Li",
        "Jianbo Yuan",
        "Xinyao Wang",
        "Qing Li",
        "Shing-Chi Cheung",
        "Shengyu Zhang",
        "Fei Wu",
        "Hongxia Yang"
      ],
      "summary": "Recent advancements in large language models (LLMs) have demonstrated substantial progress in reasoning capabilities, such as DeepSeek-R1, which leverages rule-based reinforcement learning to enhance logical reasoning significantly. However, extending these achievements to multimodal large language models (MLLMs) presents critical challenges, which are frequently more pronounced for Multimodal Small Language Models (MSLMs) given their typically weaker foundational reasoning abilities: (1) the scarcity of high-quality multimodal reasoning datasets, (2) the degradation of reasoning capabilities due to the integration of visual processing, and (3) the risk that direct application of reinforcement learning may produce complex yet incorrect reasoning processes. To address these challenges, we design a novel framework Infi-MMR to systematically unlock the reasoning potential of MSLMs through a curriculum of three carefully structured phases and propose our multimodal reasoning model Infi-MMR-3B. The first phase, Foundational Reasoning Activation, leverages high-quality textual reasoning datasets to activate and strengthen the model's logical reasoning capabilities. The second phase, Cross-Modal Reasoning Adaptation, utilizes caption-augmented multimodal data to facilitate the progressive transfer of reasoning skills to multimodal contexts. The third phase, Multimodal Reasoning Enhancement, employs curated, caption-free multimodal data to mitigate linguistic biases and promote robust cross-modal reasoning. Infi-MMR-3B achieves both state-of-the-art multimodal math reasoning ability (43.68% on MathVerse testmini, 27.04% on MathVision test, and 21.33% on OlympiadBench) and general reasoning ability (67.2% on MathVista testmini). Resources are available at https://huggingface.co/Reallm-Labs/Infi-MMR-3B.",
      "arxiv_id": "2505.23091v3",
      "published": "2025-05-29",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.23091v3",
      "pdf_url": "https://arxiv.org/pdf/2505.23091v3.pdf"
    },
    {
      "title": "RationAnomaly: Log Anomaly Detection with Rationality via Chain-of-Thought and Reinforcement Learning",
      "authors": [
        "Song Xu",
        "Yilun Liu",
        "Minggui He",
        "Mingchen Dai",
        "Ziang Chen",
        "Chunguang Zhao",
        "Jingzhou Du",
        "Shimin Tao",
        "Weibin Meng",
        "Shenglin Zhang",
        "Yongqian Sun",
        "Boxing Chen",
        "Daimeng Wei"
      ],
      "summary": "Logs constitute a form of evidence signaling the operational status of software systems. Automated log anomaly detection is crucial for ensuring the reliability of modern software systems. However, existing approaches face significant limitations: traditional deep learning models lack interpretability and generalization, while methods leveraging Large Language Models are often hindered by unreliability and factual inaccuracies. To address these issues, we propose RationAnomaly, a novel framework that enhances log anomaly detection by synergizing Chain-of-Thought (CoT) fine-tuning with reinforcement learning. Our approach first instills expert-like reasoning patterns using CoT-guided supervised fine-tuning, grounded in a high-quality dataset corrected through a rigorous expert-driven process. Subsequently, a reinforcement learning phase with a multi-faceted reward function optimizes for accuracy and logical consistency, effectively mitigating hallucinations. Experimentally, RationAnomaly outperforms state-of-the-art baselines, achieving superior F1-scores on key benchmarks while providing transparent, step-by-step analytical outputs. We have released the corresponding resources, including code and datasets.",
      "arxiv_id": "2509.14693v2",
      "published": "2025-09-18",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2509.14693v2",
      "pdf_url": "https://arxiv.org/pdf/2509.14693v2.pdf"
    },
    {
      "title": "ProRL: Prolonged Reinforcement Learning Expands Reasoning Boundaries in Large Language Models",
      "authors": [
        "Mingjie Liu",
        "Shizhe Diao",
        "Ximing Lu",
        "Jian Hu",
        "Xin Dong",
        "Yejin Choi",
        "Jan Kautz",
        "Yi Dong"
      ],
      "summary": "Recent advances in reasoning-centric language models have highlighted reinforcement learning (RL) as a promising method for aligning models with verifiable rewards. However, it remains contentious whether RL truly expands a model's reasoning capabilities or merely amplifies high-reward outputs already latent in the base model's distribution, and whether continually scaling up RL compute reliably leads to improved reasoning performance. In this work, we challenge prevailing assumptions by demonstrating that prolonged RL (ProRL) training can uncover novel reasoning strategies that are inaccessible to base models, even under extensive sampling. We introduce ProRL, a novel training methodology that incorporates KL divergence control, reference policy resetting, and a diverse suite of tasks. Our empirical analysis reveals that RL-trained models consistently outperform base models across a wide range of pass@k evaluations, including scenarios where base models fail entirely regardless of the number of attempts. We further show that reasoning boundary improvements correlates strongly with task competence of base model and training duration, suggesting that RL can explore and populate new regions of solution space over time. These findings offer new insights into the conditions under which RL meaningfully expands reasoning boundaries in language models and establish a foundation for future work on long-horizon RL for reasoning. We release model weights to support further research: https://huggingface.co/nvidia/Nemotron-Research-Reasoning-Qwen-1.5B",
      "arxiv_id": "2505.24864v1",
      "published": "2025-05-30",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.24864v1",
      "pdf_url": "https://arxiv.org/pdf/2505.24864v1.pdf"
    },
    {
      "title": "Thought Purity: A Defense Framework For Chain-of-Thought Attack",
      "authors": [
        "Zihao Xue",
        "Zhen Bi",
        "Long Ma",
        "Zhenlin Hu",
        "Yan Wang",
        "Zhenfang Liu",
        "Qing Sheng",
        "Jie Xiao",
        "Jungang Lou"
      ],
      "summary": "While reinforcement learning-trained Large Reasoning Models (LRMs, e.g., Deepseek-R1) demonstrate advanced reasoning capabilities in the evolving Large Language Models (LLMs) domain, their susceptibility to security threats remains a critical vulnerability. This weakness is particularly evident in Chain-of-Thought (CoT) generation processes, where adversarial methods like backdoor prompt attacks can systematically subvert the model's core reasoning mechanisms. The emerging Chain-of-Thought Attack (CoTA) reveals this vulnerability through exploiting prompt controllability, simultaneously degrading both CoT safety and task performance with low-cost interventions. To address this compounded security-performance vulnerability, we propose Thought Purity (TP): a defense framework that systematically strengthens resistance to malicious content while preserving operational efficacy. Our solution achieves this through three synergistic components: (1) a safety-optimized data processing pipeline (2) reinforcement learning-enhanced rule constraints (3) adaptive monitoring metrics. Our approach establishes the first comprehensive defense mechanism against CoTA vulnerabilities in reinforcement learning-aligned reasoning systems, significantly advancing the security-functionality equilibrium for next-generation AI architectures.",
      "arxiv_id": "2507.12314v2",
      "published": "2025-07-16",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CE",
        "cs.CR"
      ],
      "url": "https://arxiv.org/abs/2507.12314v2",
      "pdf_url": "https://arxiv.org/pdf/2507.12314v2.pdf"
    },
    {
      "title": "Multi-Step Reasoning with Large Language Models, a Survey",
      "authors": [
        "Aske Plaat",
        "Annie Wong",
        "Suzan Verberne",
        "Joost Broekens",
        "Niki van Stein",
        "Thomas Back"
      ],
      "summary": "Large language models (LLMs) with billions of parameters exhibit in-context learning abilities, enabling few-shot learning on tasks that the model was not specifically trained for. Traditional models achieve breakthrough performance on language tasks, but do not perform well on basic reasoning benchmarks. However, a new in-context learning approach, Chain-of-thought, has demonstrated strong multi-step reasoning abilities on these benchmarks. The research on LLM reasoning abilities started with the question whether LLMs can solve grade school math word problems, and has expanded to other tasks in the past few years. This article reviews the field of multi-step reasoning with LLMs. We propose a taxonomy that identifies different ways to generate, evaluate, and control multi-step reasoning. We provide an in-depth coverage of core approaches and open problems, and we propose a research agenda for the near future. We find that multi-step reasoning approaches have progressed beyond math word problems, and can now successfully solve challenges in logic, combinatorial games, and robotics, sometimes by first generating code that is then executed by external tools. Many studies in multi-step methods use reinforcement learning for finetuning, external optimization loops, in-context reinforcement learning, and self-reflection.",
      "arxiv_id": "2407.11511v3",
      "published": "2024-07-16",
      "categories": [
        "cs.AI",
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2407.11511v3",
      "pdf_url": "https://arxiv.org/pdf/2407.11511v3.pdf"
    },
    {
      "title": "AutoRule: Reasoning Chain-of-thought Extracted Rule-based Rewards Improve Preference Learning",
      "authors": [
        "Tevin Wang",
        "Chenyan Xiong"
      ],
      "summary": "Rule-based rewards offer a promising strategy for improving reinforcement learning from human feedback (RLHF), but current approaches often rely on manual rule engineering. We present AutoRule, a fully automated method for extracting rules from preference feedback and formulating them into rule-based rewards. AutoRule extraction operates in three stages: it leverages a reasoning model to interpret user preferences, identifies candidate rules from the reasoning chain of these interpretations, and synthesizes them into a unified rule set. Leveraging the finalized rule set, we employ language-model verifiers to compute the fraction of rules satisfied by each output, using this metric as an auxiliary reward alongside the learned reward model during policy optimization. Training a Llama-3-8B model with AutoRule results in a 28.6\\% relative improvement in length-controlled win rate on AlpacaEval2.0, and a 6.1\\% relative gain in second-turn performance on a held-out MT-Bench subset, compared to a GRPO baseline trained with the same learned reward model but without the rule-based auxiliary reward. Our analysis confirms that the extracted rules exhibit good agreement with dataset preference. We find that AutoRule demonstrates reduced reward hacking compared to a learned reward model when run over two episodes. Finally, our case study suggests that the extracted rules capture unique qualities valued in different datasets. The extracted rules are provided in the appendix, and the code is open-sourced at https://github.com/cxcscmu/AutoRule.",
      "arxiv_id": "2506.15651v1",
      "published": "2025-06-18",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.15651v1",
      "pdf_url": "https://arxiv.org/pdf/2506.15651v1.pdf"
    },
    {
      "title": "Towards Large Reasoning Models: A Survey of Reinforced Reasoning with Large Language Models",
      "authors": [
        "Fengli Xu",
        "Qianyue Hao",
        "Zefang Zong",
        "Jingwei Wang",
        "Yunke Zhang",
        "Jingyi Wang",
        "Xiaochong Lan",
        "Jiahui Gong",
        "Tianjian Ouyang",
        "Fanjin Meng",
        "Chenyang Shao",
        "Yuwei Yan",
        "Qinglong Yang",
        "Yiwen Song",
        "Sijian Ren",
        "Xinyuan Hu",
        "Yu Li",
        "Jie Feng",
        "Chen Gao",
        "Yong Li"
      ],
      "summary": "Language has long been conceived as an essential tool for human reasoning. The breakthrough of Large Language Models (LLMs) has sparked significant research interest in leveraging these models to tackle complex reasoning tasks. Researchers have moved beyond simple autoregressive token generation by introducing the concept of \"thought\" -- a sequence of tokens representing intermediate steps in the reasoning process. This innovative paradigm enables LLMs' to mimic complex human reasoning processes, such as tree search and reflective thinking. Recently, an emerging trend of learning to reason has applied reinforcement learning (RL) to train LLMs to master reasoning processes. This approach enables the automatic generation of high-quality reasoning trajectories through trial-and-error search algorithms, significantly expanding LLMs' reasoning capacity by providing substantially more training data. Furthermore, recent studies demonstrate that encouraging LLMs to \"think\" with more tokens during test-time inference can further significantly boost reasoning accuracy. Therefore, the train-time and test-time scaling combined to show a new research frontier -- a path toward Large Reasoning Model. The introduction of OpenAI's o1 series marks a significant milestone in this research direction. In this survey, we present a comprehensive review of recent progress in LLM reasoning. We begin by introducing the foundational background of LLMs and then explore the key technical components driving the development of large reasoning models, with a focus on automated data construction, learning-to-reason techniques, and test-time scaling. We also analyze popular open-source projects at building large reasoning models, and conclude with open challenges and future research directions.",
      "arxiv_id": "2501.09686v3",
      "published": "2025-01-16",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.09686v3",
      "pdf_url": "https://arxiv.org/pdf/2501.09686v3.pdf"
    },
    {
      "title": "ReAct: Synergizing Reasoning and Acting in Language Models",
      "authors": [
        "Shunyu Yao",
        "Jeffrey Zhao",
        "Dian Yu",
        "Nan Du",
        "Izhak Shafran",
        "Karthik Narasimhan",
        "Yuan Cao"
      ],
      "summary": "While large language models (LLMs) have demonstrated impressive capabilities across tasks in language understanding and interactive decision making, their abilities for reasoning (e.g. chain-of-thought prompting) and acting (e.g. action plan generation) have primarily been studied as separate topics. In this paper, we explore the use of LLMs to generate both reasoning traces and task-specific actions in an interleaved manner, allowing for greater synergy between the two: reasoning traces help the model induce, track, and update action plans as well as handle exceptions, while actions allow it to interface with external sources, such as knowledge bases or environments, to gather additional information. We apply our approach, named ReAct, to a diverse set of language and decision making tasks and demonstrate its effectiveness over state-of-the-art baselines, as well as improved human interpretability and trustworthiness over methods without reasoning or acting components. Concretely, on question answering (HotpotQA) and fact verification (Fever), ReAct overcomes issues of hallucination and error propagation prevalent in chain-of-thought reasoning by interacting with a simple Wikipedia API, and generates human-like task-solving trajectories that are more interpretable than baselines without reasoning traces. On two interactive decision making benchmarks (ALFWorld and WebShop), ReAct outperforms imitation and reinforcement learning methods by an absolute success rate of 34% and 10% respectively, while being prompted with only one or two in-context examples. Project site with code: https://react-lm.github.io",
      "arxiv_id": "2210.03629v3",
      "published": "2022-10-06",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2210.03629v3",
      "pdf_url": "https://arxiv.org/pdf/2210.03629v3.pdf"
    },
    {
      "title": "ConciseRL: Conciseness-Guided Reinforcement Learning for Efficient Reasoning Models",
      "authors": [
        "Razvan-Gabriel Dumitru",
        "Darius Peteleaza",
        "Vikas Yadav",
        "Liangming Pan"
      ],
      "summary": "Large language models excel at complex tasks by breaking down problems into structured reasoning steps. However, reasoning traces often extend beyond reaching a correct answer, causing wasted computation, reduced readability, and hallucinations. To address this, we introduce a novel hyperparameter-free conciseness score used as a reward signal within a reinforcement learning framework to guide models toward generating correct and concise reasoning traces. This score is evaluated by a large language model acting as a judge, enabling dynamic, context-aware feedback beyond simple token length. Our method achieves state-of-the-art efficiency-accuracy trade-offs on the MATH dataset, reducing token usage by up to 31x on simple problems while improving accuracy by 7%, and on the hardest problems, it outperforms full reasoning by +7.5% accuracy with up to 3.6x fewer tokens. On TheoremQA, our method improves accuracy by +2.2% using 12.5x fewer tokens. We also conduct ablation studies on the judge model, reward composition, and problem difficulty, showing that our method dynamically adapts reasoning length based on problem difficulty and benefits significantly from stronger judges. The code, model weights, and datasets are open-sourced at https://github.com/RazvanDu/ConciseRL.",
      "arxiv_id": "2505.17250v1",
      "published": "2025-05-22",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.17250v1",
      "pdf_url": "https://arxiv.org/pdf/2505.17250v1.pdf"
    },
    {
      "title": "Demystifying Long Chain-of-Thought Reasoning in LLMs",
      "authors": [
        "Edward Yeo",
        "Yuxuan Tong",
        "Morry Niu",
        "Graham Neubig",
        "Xiang Yue"
      ],
      "summary": "Scaling inference compute enhances reasoning in large language models (LLMs), with long chains-of-thought (CoTs) enabling strategies like backtracking and error correction. Reinforcement learning (RL) has emerged as a crucial method for developing these capabilities, yet the conditions under which long CoTs emerge remain unclear, and RL training requires careful design choices. In this study, we systematically investigate the mechanics of long CoT reasoning, identifying the key factors that enable models to generate long CoT trajectories. Through extensive supervised fine-tuning (SFT) and RL experiments, we present four main findings: (1) While SFT is not strictly necessary, it simplifies training and improves efficiency; (2) Reasoning capabilities tend to emerge with increased training compute, but their development is not guaranteed, making reward shaping crucial for stabilizing CoT length growth; (3) Scaling verifiable reward signals is critical for RL. We find that leveraging noisy, web-extracted solutions with filtering mechanisms shows strong potential, particularly for out-of-distribution (OOD) tasks such as STEM reasoning; and (4) Core abilities like error correction are inherently present in base models, but incentivizing these skills effectively for complex tasks via RL demands significant compute, and measuring their emergence requires a nuanced approach. These insights provide practical guidance for optimizing training strategies to enhance long CoT reasoning in LLMs. Our code is available at: https://github.com/eddycmu/demystify-long-cot.",
      "arxiv_id": "2502.03373v1",
      "published": "2025-02-05",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2502.03373v1",
      "pdf_url": "https://arxiv.org/pdf/2502.03373v1.pdf"
    },
    {
      "title": "Fortune: Formula-Driven Reinforcement Learning for Symbolic Table Reasoning in Language Models",
      "authors": [
        "Lang Cao",
        "Jingxian Xu",
        "Hanbing Liu",
        "Jinyu Wang",
        "Mengyu Zhou",
        "Haoyu Dong",
        "Shi Han",
        "Dongmei Zhang"
      ],
      "summary": "Tables are a fundamental structure for organizing and analyzing data, making effective table understanding a critical capability for intelligent systems. While large language models (LMs) demonstrate strong general reasoning abilities, they continue to struggle with accurate numerical or symbolic reasoning over tabular data, especially in complex scenarios. Spreadsheet formulas provide a powerful and expressive medium for representing executable symbolic operations, encoding rich reasoning patterns that remain largely underutilized. In this paper, we propose Formula Tuning (Fortune), a reinforcement learning (RL) framework that trains LMs to generate executable spreadsheet formulas for question answering over general tabular data. Formula Tuning reduces the reliance on supervised formula annotations by using binary answer correctness as a reward signal, guiding the model to learn formula derivation through reasoning. We provide a theoretical analysis of its advantages and demonstrate its effectiveness through extensive experiments on seven table reasoning benchmarks. Formula Tuning substantially enhances LM performance, particularly on multi-step numerical and symbolic reasoning tasks, enabling a 7B model to outperform OpenAI o1 on table understanding. This highlights the potential of formula-driven RL to advance symbolic table reasoning in LMs.",
      "arxiv_id": "2505.23667v2",
      "published": "2025-05-29",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.23667v2",
      "pdf_url": "https://arxiv.org/pdf/2505.23667v2.pdf"
    },
    {
      "title": "d1: Scaling Reasoning in Diffusion Large Language Models via Reinforcement Learning",
      "authors": [
        "Siyan Zhao",
        "Devaansh Gupta",
        "Qinqing Zheng",
        "Aditya Grover"
      ],
      "summary": "Recent large language models (LLMs) have demonstrated strong reasoning capabilities that benefits from online reinforcement learning (RL). These capabilities have primarily been demonstrated within the left-to-right autoregressive (AR) generation paradigm. In contrast, non-autoregressive paradigms based on diffusion generate text in a coarse-to-fine manner. Although recent diffusion-based large language models (dLLMs) have achieved competitive language modeling performance compared to their AR counterparts, it remains unclear if dLLMs can also leverage recent advances in LLM reasoning. To this end, we propose d1, a framework to adapt pre-trained masked dLLMs into reasoning models via a combination of supervised finetuning (SFT) and RL. Specifically, we develop and extend techniques to improve reasoning in pretrained dLLMs: (a) we utilize a masked SFT technique to distill knowledge and instill self-improvement behavior directly from existing datasets, and (b) we introduce a novel critic-free, policy-gradient based RL algorithm called diffu-GRPO, the first integration of policy gradient methods to masked dLLMs. Through empirical studies, we investigate the performance of different post-training recipes on multiple mathematical and planning benchmarks. We find that d1 yields the best performance and significantly improves performance of a state-of-the-art dLLM. Our code is released at https://dllm-reasoning.github.io/.",
      "arxiv_id": "2504.12216v2",
      "published": "2025-04-16",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2504.12216v2",
      "pdf_url": "https://arxiv.org/pdf/2504.12216v2.pdf"
    },
    {
      "title": "Adaptive Guidance Accelerates Reinforcement Learning of Reasoning Models",
      "authors": [
        "Vaskar Nath",
        "Elaine Lau",
        "Anisha Gunjal",
        "Manasi Sharma",
        "Nikhil Baharte",
        "Sean Hendryx"
      ],
      "summary": "We study the process through which reasoning models trained with reinforcement learning on verifiable rewards (RLVR) can learn to solve new problems. We find that RLVR drives performance in two main ways: (1) by compressing pass@$k$ into pass@1 and (2) via \"capability gain\" in which models learn to solve new problems that they previously could not solve even at high $k$. We find that while capability gain exists across model scales, learning to solve new problems is primarily driven through self-distillation. We demonstrate these findings across model scales ranging from 0.5B to 72B parameters on >500,000 reasoning problems with prompts and verifiable final answers across math, science, and code domains. We further show that we can significantly improve pass@$k$ rates by leveraging natural language guidance for the model to consider within context while still requiring the model to derive a solution chain from scratch. Based of these insights, we derive $\\text{Guide}$ -- a new class of online training algorithms. $\\text{Guide}$ adaptively incorporates hints into the model's context on problems for which all rollouts were initially incorrect and adjusts the importance sampling ratio for the \"off-policy\" trajectories in order to optimize the policy for contexts in which the hints are no longer present. We describe variants of $\\text{Guide}$ for GRPO and PPO and empirically show that Guide-GRPO on 7B and 32B parameter models improves generalization over its vanilla counterpart with up to 4$\\%$ macro-average improvement across math benchmarks. We include careful ablations to analyze $\\text{Guide}$'s components and theoretically analyze Guide's learning efficiency.",
      "arxiv_id": "2506.13923v2",
      "published": "2025-06-16",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.13923v2",
      "pdf_url": "https://arxiv.org/pdf/2506.13923v2.pdf"
    },
    {
      "title": "Reinforcement Learning for Reasoning in Large Language Models with One Training Example",
      "authors": [
        "Yiping Wang",
        "Qing Yang",
        "Zhiyuan Zeng",
        "Liliang Ren",
        "Liyuan Liu",
        "Baolin Peng",
        "Hao Cheng",
        "Xuehai He",
        "Kuan Wang",
        "Jianfeng Gao",
        "Weizhu Chen",
        "Shuohang Wang",
        "Simon Shaolei Du",
        "Yelong Shen"
      ],
      "summary": "We show that reinforcement learning with verifiable reward using one training example (1-shot RLVR) is effective in incentivizing the math reasoning capabilities of large language models (LLMs). Applying RLVR to the base model Qwen2.5-Math-1.5B, we identify a single example that elevates model performance on MATH500 from 36.0% to 73.6% (8.6% improvement beyond format correction), and improves the average performance across six common mathematical reasoning benchmarks from 17.6% to 35.7% (7.0% non-format gain). This result matches the performance obtained using the 1.2k DeepScaleR subset (MATH500: 73.6%, average: 35.9%), which contains the aforementioned example. Furthermore, RLVR with only two examples even slightly exceeds these results (MATH500: 74.8%, average: 36.6%). Similar substantial improvements are observed across various models (Qwen2.5-Math-7B, Llama3.2-3B-Instruct, DeepSeek-R1-Distill-Qwen-1.5B), RL algorithms (GRPO and PPO), and different math examples. In addition, we identify some interesting phenomena during 1-shot RLVR, including cross-category generalization, increased frequency of self-reflection, and sustained test performance improvement even after the training accuracy has saturated, a phenomenon we term post-saturation generalization. Moreover, we verify that the effectiveness of 1-shot RLVR primarily arises from the policy gradient loss, distinguishing it from the \"grokking\" phenomenon. We also show the critical role of promoting exploration (e.g., by incorporating entropy loss with an appropriate coefficient) in 1-shot RLVR training. We also further discuss related observations about format correction, label robustness and prompt modification. These findings can inspire future work on RLVR efficiency and encourage a re-examination of recent progress and the underlying mechanisms in RLVR. All resources are open source at https://github.com/ypwang61/One-Shot-RLVR.",
      "arxiv_id": "2504.20571v3",
      "published": "2025-04-29",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.20571v3",
      "pdf_url": "https://arxiv.org/pdf/2504.20571v3.pdf"
    },
    {
      "title": "Offline Reinforcement Learning for LLM Multi-Step Reasoning",
      "authors": [
        "Huaijie Wang",
        "Shibo Hao",
        "Hanze Dong",
        "Shenao Zhang",
        "Yilin Bao",
        "Ziran Yang",
        "Yi Wu"
      ],
      "summary": "Improving the multi-step reasoning ability of large language models (LLMs) with offline reinforcement learning (RL) is essential for quickly adapting them to complex tasks. While Direct Preference Optimization (DPO) has shown promise in aligning LLMs with human preferences, it is less suitable for multi-step reasoning tasks because (1) DPO relies on paired preference data, which is not readily available for multi-step reasoning tasks, and (2) it treats all tokens uniformly, making it ineffective for credit assignment in multi-step reasoning tasks, which often come with sparse reward. In this work, we propose OREO (Offline Reasoning Optimization), an offline RL method for enhancing LLM multi-step reasoning. Building on insights from previous works of maximum entropy reinforcement learning, it jointly learns a policy model and value function by optimizing the soft Bellman Equation. We show in principle that it reduces the need to collect pairwise data and enables better credit assignment. Empirically, OREO surpasses existing offline learning methods on multi-step reasoning benchmarks, including mathematical reasoning tasks (GSM8K, MATH) and embodied agent control (ALFWorld). The approach can be extended to a multi-iteration framework when additional resources are available. Furthermore, the learned value function can be leveraged to guide the tree search for free, which can further boost performance during test time.",
      "arxiv_id": "2412.16145v2",
      "published": "2024-12-20",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2412.16145v2",
      "pdf_url": "https://arxiv.org/pdf/2412.16145v2.pdf"
    },
    {
      "title": "Seewo's Submission to MLC-SLM: Lessons learned from Speech Reasoning Language Models",
      "authors": [
        "Bo Li",
        "Chengben Xu",
        "Wufeng Zhang"
      ],
      "summary": "This paper presents Seewo's systems for both tracks of the Multilingual Conversational Speech Language Model Challenge (MLC-SLM), addressing automatic speech recognition (ASR) and speaker diarization with ASR (SD-ASR). We introduce a multi-stage training pipeline that explicitly enhances reasoning and self-correction in speech language models for ASR. Our approach combines curriculum learning for progressive capability acquisition, Chain-of-Thought data augmentation to foster intermediate reflection, and Reinforcement Learning with Verifiable Rewards (RLVR) to further refine self-correction through reward-driven optimization. This approach achieves substantial improvements over the official challenge baselines. On the evaluation set, our best system attains a WER/CER of 11.57% for Track 1 and a tcpWER/tcpCER of 17.67% for Track 2. Comprehensive ablation studies demonstrate the effectiveness of each component under challenge constraints.",
      "arxiv_id": "2506.13300v3",
      "published": "2025-06-16",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.SD",
        "eess.AS"
      ],
      "url": "https://arxiv.org/abs/2506.13300v3",
      "pdf_url": "https://arxiv.org/pdf/2506.13300v3.pdf"
    },
    {
      "title": "Training Language Models to Reason Efficiently",
      "authors": [
        "Daman Arora",
        "Andrea Zanette"
      ],
      "summary": "Scaling model size and training data has led to great advances in the performance of Large Language Models (LLMs). However, the diminishing returns of this approach necessitate alternative methods to improve model capabilities, particularly in tasks requiring advanced reasoning. Large reasoning models, which leverage long chain-of-thoughts, bring unprecedented breakthroughs in problem-solving capabilities but at a substantial deployment cost associated to longer generations. Reducing inference costs is crucial for the economic feasibility, user experience, and environmental sustainability of these models.   In this work, we propose to train large reasoning models to reason efficiently. More precisely, we use reinforcement learning (RL) to train reasoning models to dynamically allocate inference-time compute based on task complexity. Our method incentivizes models to minimize unnecessary computational overhead while maintaining accuracy, thereby achieving substantial efficiency gains. It enables the derivation of a family of reasoning models with varying efficiency levels, controlled via a single hyperparameter. Experiments on two open-weight large reasoning models demonstrate significant reductions in inference cost while preserving most of the accuracy.",
      "arxiv_id": "2502.04463v4",
      "published": "2025-02-06",
      "categories": [
        "cs.LG",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2502.04463v4",
      "pdf_url": "https://arxiv.org/pdf/2502.04463v4.pdf"
    },
    {
      "title": "Cross-lingual Collapse: How Language-Centric Foundation Models Shape Reasoning in Large Language Models",
      "authors": [
        "Cheonbok Park",
        "Jeonghoon Kim",
        "Joosung Lee",
        "Sanghwan Bae",
        "Jaegul Choo",
        "Kang Min Yoo"
      ],
      "summary": "We identify \\textbf{Cross-lingual Collapse}, a systematic drift in which the chain-of-thought (CoT) of a multilingual language model reverts to its dominant pre-training language even when the prompt is expressed in a different language. Recent large language models (LLMs) with reinforcement learning with verifiable reward (RLVR) have achieved strong logical reasoning performances by exposing their intermediate reasoning traces, giving rise to large reasoning models (LRMs). However, the mechanism behind multilingual reasoning in LRMs is not yet fully explored. To investigate the issue, we fine-tune multilingual LRMs with Group-Relative Policy Optimization (GRPO) on translated versions of the GSM$8$K and SimpleRL-Zoo datasets in three different languages: Chinese, Korean, and Ukrainian. During training, we monitor both task accuracy and language consistency of the reasoning chains. Our experiments reveal three key findings: (i) GRPO rapidly amplifies pre-training language imbalances, leading to the erosion of low-resource languages within just a few hundred updates; (ii) language consistency reward mitigates this drift but does so at the expense of an almost 5 - 10 pp drop in accuracy. and (iii) the resulting language collapse is severely damaging and largely irreversible, as subsequent fine-tuning struggles to steer the model back toward its original target-language reasoning capabilities. Together, these findings point to a remarkable conclusion: \\textit{not all languages are trained equally for reasoning}. Furthermore, our paper sheds light on the roles of reward shaping, data difficulty, and pre-training priors in eliciting multilingual reasoning.",
      "arxiv_id": "2506.05850v2",
      "published": "2025-06-06",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.05850v2",
      "pdf_url": "https://arxiv.org/pdf/2506.05850v2.pdf"
    },
    {
      "title": "RL of Thoughts: Navigating LLM Reasoning with Inference-time Reinforcement Learning",
      "authors": [
        "Qianyue Hao",
        "Sibo Li",
        "Jian Yuan",
        "Yong Li"
      ],
      "summary": "Despite rapid advancements in large language models (LLMs), the token-level autoregressive nature constrains their complex reasoning capabilities. To enhance LLM reasoning, inference-time techniques, including Chain/Tree/Graph-of-Thought(s), successfully improve the performance, as they are fairly cost-effective by guiding reasoning through sophisticated logical structures without modifying LLMs' parameters. However, these manually predefined, task-agnostic frameworks are applied uniformly across diverse tasks, lacking adaptability. To improve this, we propose RL-of-Thoughts (RLoT), where we train a lightweight navigator model with reinforcement learning (RL) to adaptively enhance LLM reasoning at inference time. Specifically, we design five basic logic blocks from the perspective of human cognition. During the reasoning process, the trained RL navigator dynamically selects the suitable logic blocks and combines them into task-specific logical structures according to problem characteristics. Experiments across multiple reasoning benchmarks (AIME, MATH, GPQA, etc.) with multiple LLMs (GPT, Llama, Qwen, and DeepSeek) illustrate that RLoT outperforms established inference-time techniques by up to 13.4%. Remarkably, with less than 3K parameters, our RL navigator is able to make sub-10B LLMs comparable to 100B-scale counterparts. Moreover, the RL navigator demonstrates strong transferability: a model trained on one specific LLM-task pair can effectively generalize to unseen LLMs and tasks. Our code is open-source at https://anonymous.4open.science/r/RL-LLM-Reasoning-1A30 for reproducibility.",
      "arxiv_id": "2505.14140v2",
      "published": "2025-05-20",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.14140v2",
      "pdf_url": "https://arxiv.org/pdf/2505.14140v2.pdf"
    },
    {
      "title": "Reasoning through Exploration: A Reinforcement Learning Framework for Robust Function Calling",
      "authors": [
        "Bingguang Hao",
        "Zengzhuang Xu",
        "Maolin Wang",
        "Yuntao Wen",
        "Yicheng Chen",
        "Cunyin Peng",
        "Long Chen",
        "Dong Wang",
        "Xiangyu Zhao",
        "Jinjie Gu",
        "Chenyi Zhuang",
        "Ji Zhang"
      ],
      "summary": "The effective training of Large Language Models (LLMs) for function calling faces a critical challenge: balancing exploration of complex reasoning paths with stable policy optimization. Standard methods like Supervised Fine-Tuning (SFT) fail to instill robust reasoning, and traditional Reinforcement Learning (RL) struggles with inefficient exploration. We propose \\textbf{EGPO}, a new RL framework built upon Group Relative Policy Optimization (GRPO), designed to address this challenge directly. The core of EGPO is an entropy-enhanced advantage function that integrates the entropy of the model's Chain-of-Thought (CoT) into the policy gradient computation. This encourages the generation of diverse reasoning strategies. To maintain optimization direction, the entropy bonus is carefully constrained by a clipping mechanism. Complemented by a strict, binary reward signal, EGPO effectively guides the model towards discovering structured and accurate tool invocation patterns. On the challenging Berkeley Function Calling Leaderboard (BFCL), a 4B-parameter model trained with EGPO sets a new state-of-the-art among models of comparable size, surpassing a range of strong competitors, including GPT-4o and Gemini-2.5.",
      "arxiv_id": "2508.05118v4",
      "published": "2025-08-07",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2508.05118v4",
      "pdf_url": "https://arxiv.org/pdf/2508.05118v4.pdf"
    },
    {
      "title": "Emma-X: An Embodied Multimodal Action Model with Grounded Chain of Thought and Look-ahead Spatial Reasoning",
      "authors": [
        "Qi Sun",
        "Pengfei Hong",
        "Tej Deep Pala",
        "Vernon Toh",
        "U-Xuan Tan",
        "Deepanway Ghosal",
        "Soujanya Poria"
      ],
      "summary": "Traditional reinforcement learning-based robotic control methods are often task-specific and fail to generalize across diverse environments or unseen objects and instructions. Visual Language Models (VLMs) demonstrate strong scene understanding and planning capabilities but lack the ability to generate actionable policies tailored to specific robotic embodiments. To address this, Visual-Language-Action (VLA) models have emerged, yet they face challenges in long-horizon spatial reasoning and grounded task planning. In this work, we propose the Embodied Multimodal Action Model with Grounded Chain of Thought and Look-ahead Spatial Reasoning, Emma-X. Emma-X leverages our constructed hierarchical embodiment dataset based on BridgeV2, containing 60,000 robot manipulation trajectories auto-annotated with grounded task reasoning and spatial guidance. Additionally, we introduce a trajectory segmentation strategy based on gripper states and motion trajectories, which can help mitigate hallucination in grounding subtask reasoning generation. Experimental results demonstrate that Emma-X achieves superior performance over competitive baselines, particularly in real-world robotic tasks requiring spatial reasoning.",
      "arxiv_id": "2412.11974v2",
      "published": "2024-12-16",
      "categories": [
        "cs.RO",
        "cs.AI",
        "cs.CL",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2412.11974v2",
      "pdf_url": "https://arxiv.org/pdf/2412.11974v2.pdf"
    },
    {
      "title": "Adaptive Graph of Thoughts: Test-Time Adaptive Reasoning Unifying Chain, Tree, and Graph Structures",
      "authors": [
        "Tushar Pandey",
        "Ara Ghukasyan",
        "Oktay Goktas",
        "Santosh Kumar Radha"
      ],
      "summary": "Large Language Models (LLMs) have demonstrated impressive reasoning capabilities, yet their performance is highly dependent on the prompting strategy and model scale. While reinforcement learning and fine-tuning have been deployed to boost reasoning, these approaches incur substantial computational and data overhead. In this work, we introduce Adaptive Graph of Thoughts (AGoT), a dynamic, graph-based inference framework that enhances LLM reasoning solely at test time. Rather than relying on fixed-step methods like Chain of Thought (CoT) or Tree of Thoughts (ToT), AGoT recursively decomposes complex queries into structured subproblems, forming an dynamic directed acyclic graph (DAG) of interdependent reasoning steps. By selectively expanding only those subproblems that require further analysis, AGoT unifies the strengths of chain, tree, and graph paradigms into a cohesive framework that allocates computation where it is most needed. We validate our approach on diverse benchmarks spanning multi-hop retrieval, scientific reasoning, and mathematical problem-solving, achieving up to 46.2% improvement on scientific reasoning tasks (GPQA) - comparable to gains achieved through computationally intensive reinforcement learning approaches and outperforming state-of-the-art iterative approaches. These results suggest that dynamic decomposition and structured recursion offer a scalable, cost-effective alternative to post-training modifications, paving the way for more robust, general-purpose reasoning in LLMs.",
      "arxiv_id": "2502.05078v1",
      "published": "2025-02-07",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2502.05078v1",
      "pdf_url": "https://arxiv.org/pdf/2502.05078v1.pdf"
    },
    {
      "title": "ReasonFlux-PRM: Trajectory-Aware PRMs for Long Chain-of-Thought Reasoning in LLMs",
      "authors": [
        "Jiaru Zou",
        "Ling Yang",
        "Jingwen Gu",
        "Jiahao Qiu",
        "Ke Shen",
        "Jingrui He",
        "Mengdi Wang"
      ],
      "summary": "Process Reward Models (PRMs) have recently emerged as a powerful framework for supervising intermediate reasoning steps in large language models (LLMs). Previous PRMs are primarily trained on model final output responses and struggle to evaluate intermediate thinking trajectories robustly, especially in the emerging setting of trajectory-response outputs generated by frontier reasoning models like Deepseek-R1. In this work, we introduce ReasonFlux-PRM, a novel trajectory-aware PRM explicitly designed to evaluate the trajectory-response type of reasoning traces. ReasonFlux-PRM incorporates both step-level and trajectory-level supervision, enabling fine-grained reward assignment aligned with structured chain-of-thought data. We adapt ReasonFlux-PRM to support reward supervision under both offline and online settings, including (i) selecting high-quality model distillation data for downstream supervised fine-tuning of smaller models, (ii) providing dense process-level rewards for policy optimization during reinforcement learning, and (iii) enabling reward-guided Best-of-N test-time scaling. Empirical results on challenging downstream benchmarks such as AIME, MATH500, and GPQA-Diamond demonstrate that ReasonFlux-PRM-7B selects higher quality data than strong PRMs (e.g., Qwen2.5-Math-PRM-72B) and human-curated baselines. Furthermore, our derived ReasonFlux-PRM-7B yields consistent performance improvements, achieving average gains of 12.1% in supervised fine-tuning, 4.5% in reinforcement learning, and 6.3% in test-time scaling. We also release our efficient ReasonFlux-PRM-1.5B for resource-constrained applications and edge deployment. Project: https://github.com/Gen-Verse/ReasonFlux",
      "arxiv_id": "2506.18896v2",
      "published": "2025-06-23",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.18896v2",
      "pdf_url": "https://arxiv.org/pdf/2506.18896v2.pdf"
    },
    {
      "title": "Think in Games: Learning to Reason in Games via Reinforcement Learning with Large Language Models",
      "authors": [
        "Yi Liao",
        "Yu Gu",
        "Yuan Sui",
        "Zining Zhu",
        "Yifan Lu",
        "Guohua Tang",
        "Zhongqian Sun",
        "Wei Yang"
      ],
      "summary": "Large language models (LLMs) excel at complex reasoning tasks such as mathematics and coding, yet they frequently struggle with simple interactive tasks that young children perform effortlessly. This discrepancy highlights a critical gap between declarative knowledge (knowing about something) and procedural knowledge (knowing how to do something). Although traditional reinforcement learning (RL) agents can acquire procedural knowledge through environmental interaction, they often operate as black boxes and require substantial training data. In contrast, LLMs possess extensive world knowledge and reasoning capabilities, but are unable to effectively convert this static knowledge into dynamic decision-making in interactive settings. To address this challenge, we propose Think in Games (TiG), a novel framework that empowers LLMs to develop procedural understanding through direct interaction with game environments, while retaining their inherent reasoning and explanatory abilities. Specifically, TiG reformulates RL-based decision-making as a language modeling task: LLMs generate language-guided policies, which are refined iteratively through online reinforcement learning based on environmental feedback. Our experimental results show that TiG successfully bridges the gap between declarative and procedural knowledge, achieving competitive performance with dramatically lower data and computational demands compared to conventional RL methods. Moreover, TiG provides step-by-step natural language explanations for its decisions, greatly improving transparency and interpretability in complex interactive tasks.",
      "arxiv_id": "2508.21365v1",
      "published": "2025-08-29",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.21365v1",
      "pdf_url": "https://arxiv.org/pdf/2508.21365v1.pdf"
    },
    {
      "title": "h1: Bootstrapping LLMs to Reason over Longer Horizons via Reinforcement Learning",
      "authors": [
        "Sumeet Ramesh Motwani",
        "Alesia Ivanova",
        "Ziyang Cai",
        "Philip Torr",
        "Riashat Islam",
        "Shital Shah",
        "Christian Schroeder de Witt",
        "Charles London"
      ],
      "summary": "Large language models excel at short-horizon reasoning tasks, but performance drops as reasoning horizon lengths increase. Existing approaches to combat this rely on inference-time scaffolding or costly step-level supervision, neither of which scales easily. In this work, we introduce a scalable method to bootstrap long-horizon reasoning capabilities using only existing, abundant short-horizon data. Our approach synthetically composes simple problems into complex, multi-step dependency chains of arbitrary length. We train models on this data using outcome-only rewards under a curriculum that automatically increases in complexity, allowing RL training to be scaled much further without saturating. Empirically, our method generalizes remarkably well: curriculum training on composed 6th-grade level math problems (GSM8K) boosts accuracy on longer, competition-level benchmarks (GSM-Symbolic, MATH-500, AIME) by up to 2.06x. It also transfers significantly to diverse out-of-distribution ReasoningGym domains and long-context benchmarks, indicating broader generalization. Importantly, our long-horizon improvements are significantly higher than baselines even at high pass@k, showing that models can learn new reasoning paths under RL. Theoretically, we show that curriculum RL with outcome rewards achieves an exponential improvement in sample complexity over full-horizon training, providing training signal comparable to dense supervision. h1 therefore introduces an efficient path towards scaling RL for long-horizon problems using only existing data.",
      "arxiv_id": "2510.07312v2",
      "published": "2025-10-08",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.07312v2",
      "pdf_url": "https://arxiv.org/pdf/2510.07312v2.pdf"
    },
    {
      "title": "Towards Reasoning Era: A Survey of Long Chain-of-Thought for Reasoning Large Language Models",
      "authors": [
        "Qiguang Chen",
        "Libo Qin",
        "Jinhao Liu",
        "Dengyun Peng",
        "Jiannan Guan",
        "Peng Wang",
        "Mengkang Hu",
        "Yuhang Zhou",
        "Te Gao",
        "Wanxiang Che"
      ],
      "summary": "Recent advancements in reasoning with large language models (RLLMs), such as OpenAI-O1 and DeepSeek-R1, have demonstrated their impressive capabilities in complex domains like mathematics and coding. A central factor in their success lies in the application of long chain-of-thought (Long CoT) characteristics, which enhance reasoning abilities and enable the solution of intricate problems. However, despite these developments, a comprehensive survey on Long CoT is still lacking, limiting our understanding of its distinctions from traditional short chain-of-thought (Short CoT) and complicating ongoing debates on issues like \"overthinking\" and \"inference-time scaling.\" This survey seeks to fill this gap by offering a unified perspective on Long CoT. (1) We first distinguish Long CoT from Short CoT and introduce a novel taxonomy to categorize current reasoning paradigms. (2) Next, we explore the key characteristics of Long CoT: deep reasoning, extensive exploration, and feasible reflection, which enable models to handle more complex tasks and produce more efficient, coherent outcomes compared to the shallower Short CoT. (3) We then investigate key phenomena such as the emergence of Long CoT with these characteristics, including overthinking, and inference-time scaling, offering insights into how these processes manifest in practice. (4) Finally, we identify significant research gaps and highlight promising future directions, including the integration of multi-modal reasoning, efficiency improvements, and enhanced knowledge frameworks. By providing a structured overview, this survey aims to inspire future research and further the development of logical reasoning in artificial intelligence.",
      "arxiv_id": "2503.09567v5",
      "published": "2025-03-12",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2503.09567v5",
      "pdf_url": "https://arxiv.org/pdf/2503.09567v5.pdf"
    },
    {
      "title": "Nemotron-Research-Tool-N1: Exploring Tool-Using Language Models with Reinforced Reasoning",
      "authors": [
        "Shaokun Zhang",
        "Yi Dong",
        "Jieyu Zhang",
        "Jan Kautz",
        "Bryan Catanzaro",
        "Andrew Tao",
        "Qingyun Wu",
        "Zhiding Yu",
        "Guilin Liu"
      ],
      "summary": "Enabling large language models with external tools has become a pivotal strategy for extending their functionality beyond text space. To enhance LLMs' tool-calling abilities, previous approaches primarily rely on supervised fine-tuning (SFT) with trajectories distilled from stronger models, often resulting in imitative reasoning that limits generalization. In this work, we explore rule-based reinforcement learning to enhance tool-calling in LLMs, resulting in Nemotron-Research-Tool-N1, a series of tool-calling reasoning models. Rather than enforcing supervision over intermediate distilled reasoning traces, Tool-N1 is trained with a binary RL reward that assesses only the format validity and functional correctness of tool invocations. This lightweight supervision allows the model to develop reasoning strategies independently, without relying on annotated trajectories. Experiments on several major benchmarks show that Tool-N1-7B/14B clearly outperform GPT-4o. We conduct a systematic study on the design of rule-based reinforcement learning strategies for training tool-calling models. Using 5,518 distilled reasoning trajectories, we compare SFT, RL, and the SFT-then-RL pipeline, finding that the widely adopted SFT-then-RL paradigm does not necessarily outperform pure RL.",
      "arxiv_id": "2505.00024v2",
      "published": "2025-04-25",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.00024v2",
      "pdf_url": "https://arxiv.org/pdf/2505.00024v2.pdf"
    },
    {
      "title": "Optimizing Chain-of-Thought Reasoners via Gradient Variance Minimization in Rejection Sampling and RL",
      "authors": [
        "Jiarui Yao",
        "Yifan Hao",
        "Hanning Zhang",
        "Hanze Dong",
        "Wei Xiong",
        "Nan Jiang",
        "Tong Zhang"
      ],
      "summary": "Chain-of-thought (CoT) reasoning in large language models (LLMs) can be formalized as a latent variable problem, where the model needs to generate intermediate reasoning steps. While prior approaches such as iterative reward-ranked fine-tuning (RAFT) have relied on such formulations, they typically apply uniform inference budgets across prompts, which fails to account for variability in difficulty and convergence behavior. This work identifies the main bottleneck in CoT training as inefficient stochastic gradient estimation due to static sampling strategies. We propose GVM-RAFT, a prompt-specific Dynamic Sample Allocation Strategy designed to minimize stochastic gradient variance under a computational budget constraint. The method dynamically allocates computational resources by monitoring prompt acceptance rates and stochastic gradient norms, ensuring that the resulting gradient variance is minimized. Our theoretical analysis shows that the proposed dynamic sampling strategy leads to accelerated convergence guarantees under suitable conditions. Experiments on mathematical reasoning show that GVM-RAFT achieves a 2-4x speedup and considerable accuracy improvements over vanilla RAFT. The proposed dynamic sampling strategy is general and can be incorporated into other reinforcement learning algorithms, such as GRPO, leading to similar improvements in convergence and test accuracy. Our code is available at https://github.com/RLHFlow/GVM.",
      "arxiv_id": "2505.02391v1",
      "published": "2025-05-05",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.02391v1",
      "pdf_url": "https://arxiv.org/pdf/2505.02391v1.pdf"
    },
    {
      "title": "Learning a Dense Reasoning Reward Model from Expert Demonstration via Inverse Reinforcement Learning",
      "authors": [
        "Claudio Fanconi",
        "Nicol\u00e1s Astorga",
        "Mihaela van der Schaar"
      ],
      "summary": "We reframe and operationalise adversarial inverse reinforcement learning (IRL) to large language model reasoning, learning a dense, token-level reward model for process supervision directly from expert demonstrations rather than imitating style via supervised fine-tuning. The learned reasoning reward serves two complementary roles: (i) it provides step-level feedback to optimise a reasoning policy during training; and (ii) it functions at inference as a critic to rerank sampled traces under fixed compute budgets. We demonstrate that our approach prioritises correctness over surface form, yielding scores that correlate with eventual answer validity and enabling interpretable localisation of errors within a trace. Empirically, on GSM8K with Llama3 and Qwen2.5 backbones, we demonstrate: (i) dense reasoning rewards can be used as a learning signal to elicit reasoning, and (ii) predictive performance is improved from reward-guided reranking (notably for Llama-based policies). By unifying training signals, inference-time selection, and token-level diagnostics into a single reasoning reward, this work suggests reusable process-level rewards with broad potential to enhance multi-step reasoning in language models.",
      "arxiv_id": "2510.01857v1",
      "published": "2025-10-02",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.01857v1",
      "pdf_url": "https://arxiv.org/pdf/2510.01857v1.pdf"
    },
    {
      "title": "Teaching AI Stepwise Diagnostic Reasoning with Report-Guided Chain-of-Thought Learning",
      "authors": [
        "Yihong Luo",
        "Wenwu He",
        "Zhuo-Xu Cui",
        "Dong Liang"
      ],
      "summary": "This study presents DiagCoT, a multi-stage framework that applies supervised fine-tuning to general-purpose vision-language models (VLMs) to emulate radiologists' stepwise diagnostic reasoning using only free-text reports. DiagCoT combines contrastive image-report tuning for domain alignment, chain-of-thought supervision to capture inferential logic, and reinforcement tuning with clinical reward signals to enhance factual accuracy and fluency. On the MIMIC-CXR benchmark, DiagCoT improved zero-shot disease classification AUC from 0.52 to 0.76 (absolute gain of 0.24), pathology grounding mIoU from 0.08 to 0.31 (absolute gain of 0.23), and report generation BLEU from 0.11 to 0.33 (absolute gain of 0.22). It outperformed state-of-the-art models including LLaVA-Med and CXR-LLAVA on long-tailed diseases and external datasets. By converting unstructured clinical narratives into structured supervision, DiagCoT offers a scalable approach for developing interpretable and diagnostically competent AI systems for radiology.",
      "arxiv_id": "2509.06409v1",
      "published": "2025-09-08",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2509.06409v1",
      "pdf_url": "https://arxiv.org/pdf/2509.06409v1.pdf"
    },
    {
      "title": "Retrospex: Language Agent Meets Offline Reinforcement Learning Critic",
      "authors": [
        "Yufei Xiang",
        "Yiqun Shen",
        "Yeqin Zhang",
        "Cam-Tu Nguyen"
      ],
      "summary": "Large Language Models (LLMs) possess extensive knowledge and commonsense reasoning capabilities, making them valuable for creating powerful agents. However, existing LLM agent frameworks have not fully utilized past experiences for improvement. This work introduces a new LLM-based agent framework called Retrospex, which addresses this challenge by analyzing past experiences in depth. Unlike previous approaches, Retrospex does not directly integrate experiences into the LLM's context. Instead, it combines the LLM's action likelihood with action values estimated by a Reinforcement Learning (RL) Critic, which is trained on past experiences through an offline ''retrospection'' process. Additionally, Retrospex employs a dynamic action rescoring mechanism that increases the importance of experience-based values for tasks that require more interaction with the environment. We evaluate Retrospex in ScienceWorld, ALFWorld and Webshop environments, demonstrating its advantages over strong, contemporary baselines.",
      "arxiv_id": "2505.11807v2",
      "published": "2025-05-17",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.11807v2",
      "pdf_url": "https://arxiv.org/pdf/2505.11807v2.pdf"
    },
    {
      "title": "Narrative-Guided Reinforcement Learning: A Platform for Studying Language Model Influence on Decision Making",
      "authors": [
        "Anup Tuladhar",
        "Araz Minhas",
        "Adam Kirton",
        "Eli Kinney-Lang"
      ],
      "summary": "We present a preliminary experimental platform that explores how narrative elements might shape AI decision-making by combining reinforcement learning (RL) with language model reasoning. While AI systems can now both make decisions and engage in narrative reasoning, these capabilities have mostly been studied separately. Our platform attempts to bridge this gap using a dual-system architecture to examine how narrative frameworks could influence reward-based learning. The system comprises a reinforcement learning policy that suggests actions based on past experience, and a language model that processes these suggestions through different narrative frameworks to guide decisions. This setup enables initial experimentation with narrative elements while maintaining consistent environment and reward structures. We implement this architecture in a configurable gridworld environment, where agents receive both policy suggestions and information about their surroundings. The platform's modular design facilitates controlled testing of environmental complexity, narrative parameters, and the interaction between reinforcement learning and narrative-based decisions. Our logging system captures basic decision metrics, from RL policy values to language model reasoning to action selection patterns. While preliminary, this implementation provides a foundation for studying how different narrative frameworks might affect reward-based decisions and exploring potential interactions between optimization-based learning and symbolic reasoning in AI systems.",
      "arxiv_id": "2509.08785v1",
      "published": "2025-09-10",
      "categories": [
        "cs.AI",
        "cs.MA",
        "stat.ML"
      ],
      "url": "https://arxiv.org/abs/2509.08785v1",
      "pdf_url": "https://arxiv.org/pdf/2509.08785v1.pdf"
    },
    {
      "title": "BRiTE: Bootstrapping Reinforced Thinking Process to Enhance Language Model Reasoning",
      "authors": [
        "Han Zhong",
        "Yutong Yin",
        "Shenao Zhang",
        "Xiaojun Xu",
        "Yuanxin Liu",
        "Yifei Zuo",
        "Zhihan Liu",
        "Boyi Liu",
        "Sirui Zheng",
        "Hongyi Guo",
        "Liwei Wang",
        "Mingyi Hong",
        "Zhaoran Wang"
      ],
      "summary": "Large Language Models (LLMs) have demonstrated remarkable capabilities in complex reasoning tasks, yet generating reliable reasoning processes remains a significant challenge. We present a unified probabilistic framework that formalizes LLM reasoning through a novel graphical model incorporating latent thinking processes and evaluation signals. Within this framework, we introduce the Bootstrapping Reinforced Thinking Process (BRiTE) algorithm, which works in two steps. First, it generates high-quality rationales by approximating the optimal thinking process through reinforcement learning, using a novel reward shaping mechanism. Second, it enhances the base LLM by maximizing the joint probability of rationale generation with respect to the model's parameters. Theoretically, we demonstrate BRiTE's convergence at a rate of $1/T$ with $T$ representing the number of iterations. Empirical evaluations on math and coding benchmarks demonstrate that our approach consistently improves performance across different base models without requiring human-annotated thinking processes. In addition, BRiTE demonstrates superior performance compared to existing algorithms that bootstrap thinking processes use alternative methods such as rejection sampling, and can even match or exceed the results achieved through supervised fine-tuning with human-annotated data.",
      "arxiv_id": "2501.18858v2",
      "published": "2025-01-31",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.18858v2",
      "pdf_url": "https://arxiv.org/pdf/2501.18858v2.pdf"
    },
    {
      "title": "PeRL: Permutation-Enhanced Reinforcement Learning for Interleaved Vision-Language Reasoning",
      "authors": [
        "Yizhen Zhang",
        "Yang Ding",
        "Shuoshuo Zhang",
        "Xinchen Zhang",
        "Haoling Li",
        "Zhong-zhi Li",
        "Peijie Wang",
        "Jie Wu",
        "Lei Ji",
        "Yelong Shen",
        "Yujiu Yang",
        "Yeyun Gong"
      ],
      "summary": "Inspired by the impressive reasoning capabilities demonstrated by reinforcement learning approaches like DeepSeek-R1, recent emerging research has begun exploring the use of reinforcement learning (RL) to enhance vision-language models (VLMs) for multimodal reasoning tasks. However, most existing multimodal reinforcement learning approaches remain limited to spatial reasoning within single-image contexts, yet still struggle to generalize to more complex and real-world scenarios involving multi-image positional reasoning, where understanding the relationships across images is crucial. To address this challenge, we propose a general reinforcement learning approach PeRL tailored for interleaved multimodal tasks, and a multi-stage strategy designed to enhance the exploration-exploitation trade-off, thereby improving learning efficiency and task performance. Specifically, we introduce permutation of image sequences to simulate varied positional relationships to explore more spatial and positional diversity. Furthermore, we design a rollout filtering mechanism for resampling to focus on trajectories that contribute most to learning optimal behaviors to exploit learned policies effectively. We evaluate our model on 5 widely-used multi-image benchmarks and 3 single-image benchmarks. Our experiments confirm that PeRL trained model consistently surpasses R1-related and interleaved VLM baselines by a large margin, achieving state-of-the-art performance on multi-image benchmarks, while preserving comparable performance on single-image tasks.",
      "arxiv_id": "2506.14907v1",
      "published": "2025-06-17",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.14907v1",
      "pdf_url": "https://arxiv.org/pdf/2506.14907v1.pdf"
    },
    {
      "title": "Ego-R1: Chain-of-Tool-Thought for Ultra-Long Egocentric Video Reasoning",
      "authors": [
        "Shulin Tian",
        "Ruiqi Wang",
        "Hongming Guo",
        "Penghao Wu",
        "Yuhao Dong",
        "Xiuying Wang",
        "Jingkang Yang",
        "Hao Zhang",
        "Hongyuan Zhu",
        "Ziwei Liu"
      ],
      "summary": "We introduce Ego-R1, a novel framework for reasoning over ultra-long (i.e., in days and weeks) egocentric videos, which leverages a structured Chain-of-Tool-Thought (CoTT) process, orchestrated by an Ego-R1 Agent trained via reinforcement learning (RL). Inspired by human problem-solving strategies, CoTT decomposes complex reasoning into modular steps, with the RL agent invoking specific tools, one per step, to iteratively and collaboratively answer sub-questions tackling such tasks as temporal retrieval and multi-modal understanding. We design a two-stage training paradigm involving supervised finetuning (SFT) of a pretrained language model using CoTT data and RL to enable our agent to dynamically propose step-by-step tools for long-range reasoning. To facilitate training, we construct a dataset called Ego-R1 Data, which consists of Ego-CoTT-25K for SFT and Ego-QA-4.4K for RL. Furthermore, our Ego-R1 agent is evaluated on a newly curated week-long video QA benchmark, Ego-R1 Bench, which contains human-verified QA pairs from hybrid sources. Extensive results demonstrate that the dynamic, tool-augmented chain-of-thought reasoning by our Ego-R1 Agent can effectively tackle the unique challenges of understanding ultra-long egocentric videos, significantly extending the time coverage from few hours to a week.",
      "arxiv_id": "2506.13654v1",
      "published": "2025-06-16",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.13654v1",
      "pdf_url": "https://arxiv.org/pdf/2506.13654v1.pdf"
    },
    {
      "title": "Mind the Gap: Bridging Thought Leap for Improved Chain-of-Thought Tuning",
      "authors": [
        "Haolei Xu",
        "Yuchen Yan",
        "Yongliang Shen",
        "Wenqi Zhang",
        "Guiyang Hou",
        "Shengpei Jiang",
        "Kaitao Song",
        "Weiming Lu",
        "Jun Xiao",
        "Yueting Zhuang"
      ],
      "summary": "Large language models (LLMs) have achieved remarkable progress on mathematical tasks through Chain-of-Thought (CoT) reasoning. However, existing mathematical CoT datasets often suffer from Thought Leaps due to experts omitting intermediate steps, which negatively impacts model learning and generalization. We propose the CoT Thought Leap Bridge Task, which aims to automatically detect leaps and generate missing intermediate reasoning steps to restore the completeness and coherence of CoT. To facilitate this, we constructed a specialized training dataset called ScaleQM+, based on the structured ScaleQuestMath dataset, and trained CoT-Bridge to bridge thought leaps. Through comprehensive experiments on mathematical reasoning benchmarks, we demonstrate that models fine-tuned on bridged datasets consistently outperform those trained on original datasets, with improvements of up to +5.87% on NuminaMath. Our approach effectively enhances distilled data (+3.02%) and provides better starting points for reinforcement learning (+3.1%), functioning as a plug-and-play module compatible with existing optimization techniques. Furthermore, CoT-Bridge demonstrate improved generalization to out-of-domain logical reasoning tasks, confirming that enhancing reasoning completeness yields broadly applicable benefits.",
      "arxiv_id": "2505.14684v2",
      "published": "2025-05-20",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.14684v2",
      "pdf_url": "https://arxiv.org/pdf/2505.14684v2.pdf"
    },
    {
      "title": "R2Vul: Learning to Reason about Software Vulnerabilities with Reinforcement Learning and Structured Reasoning Distillation",
      "authors": [
        "Martin Weyssow",
        "Chengran Yang",
        "Junkai Chen",
        "Ratnadira Widyasari",
        "Ting Zhang",
        "Huihui Huang",
        "Huu Hung Nguyen",
        "Yan Naing Tun",
        "Tan Bui",
        "Yikun Li",
        "Ang Han Wei",
        "Frank Liauw",
        "Eng Lieh Ouh",
        "Lwin Khin Shar",
        "David Lo"
      ],
      "summary": "Large language models (LLMs) have shown promising performance in software vulnerability detection, yet their reasoning capabilities remain unreliable. We propose R2Vul, a method that combines reinforcement learning from AI feedback (RLAIF) and structured reasoning distillation to teach small code LLMs to detect vulnerabilities while generating security-aware explanations. Unlike prior chain-of-thought and instruction tuning approaches, R2Vul rewards well-founded over deceptively plausible vulnerability explanations through RLAIF, which results in more precise detection and high-quality reasoning generation. To support RLAIF, we construct the first multilingual preference dataset for vulnerability detection, comprising 18,000 high-quality samples in C\\#, JavaScript, Java, Python, and C. We evaluate R2Vul across five programming languages and against four static analysis tools, eight state-of-the-art LLM-based baselines, and various fine-tuning approaches. Our results demonstrate that a 1.5B R2Vul model exceeds the performance of its 32B teacher model and leading commercial LLMs such as Claude-4-Opus. Furthermore, we introduce a lightweight calibration step that reduces false positive rates under varying imbalanced data distributions. Finally, through qualitative analysis, we show that both LLM and human evaluators consistently rank R2Vul model's reasoning higher than other reasoning-based baselines.",
      "arxiv_id": "2504.04699v2",
      "published": "2025-04-07",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.04699v2",
      "pdf_url": "https://arxiv.org/pdf/2504.04699v2.pdf"
    },
    {
      "title": "WizardMath: Empowering Mathematical Reasoning for Large Language Models via Reinforced Evol-Instruct",
      "authors": [
        "Haipeng Luo",
        "Qingfeng Sun",
        "Can Xu",
        "Pu Zhao",
        "Jianguang Lou",
        "Chongyang Tao",
        "Xiubo Geng",
        "Qingwei Lin",
        "Shifeng Chen",
        "Yansong Tang",
        "Dongmei Zhang"
      ],
      "summary": "Large language models (LLMs), such as GPT-4, have shown remarkable performance in natural language processing (NLP) tasks, including challenging mathematical reasoning. However, most existing open-source models are only pre-trained on large-scale internet data and without math-related optimization. In this paper, we present WizardMath, which enhances the mathematical CoT reasoning abilities of LLMs without using external python tools, by applying our proposed Reinforcement Learning from Evol-Instruct Feedback (RLEIF) method to the domain of math. Through extensive experiments on two mathematical reasoning benchmarks, namely GSM8k and MATH, we reveal the extraordinary capabilities of our model. Remarkably, WizardMath-Mistral 7B surpasses top-tier open-source LLMs by a substantial margin with higher data efficiency. Furthermore, WizardMath 70B even outperforms GPT-3.5-Turbo, Claude 2, Gemini Pro and GPT-4-early-version. Additionally, our preliminary exploration highlights the pivotal role of instruction evolution and process supervision in achieving exceptional math performance. For more details refer to https://github.com/nlpxucan/WizardLM",
      "arxiv_id": "2308.09583v3",
      "published": "2023-08-18",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2308.09583v3",
      "pdf_url": "https://arxiv.org/pdf/2308.09583v3.pdf"
    },
    {
      "title": "Post-Training Large Language Models via Reinforcement Learning from Self-Feedback",
      "authors": [
        "Carel van Niekerk",
        "Renato Vukovic",
        "Benjamin Matthias Ruppik",
        "Hsien-chin Lin",
        "Milica Ga\u0161i\u0107"
      ],
      "summary": "Large Language Models (LLMs) often produce plausible but poorly-calibrated answers, limiting their reliability on reasoning-intensive tasks. We present Reinforcement Learning from Self-Feedback (RLSF), a post-training stage that uses the model's own confidence as an intrinsic reward, mimicking how humans learn in the absence of external feedback. After a frozen LLM generates several chain-of-thought solutions, we define and compute the confidence of each final answer span and rank the traces accordingly. These synthetic preferences are then used to fine-tune the policy with standard preference optimization, similar to RLHF yet requiring no human labels, gold answers, or externally curated rewards.   RLSF simultaneously (i) refines the model's probability estimates -- restoring well-behaved calibration -- and (ii) strengthens step-by-step reasoning, yielding improved performance on arithmetic reasoning and multiple-choice question answering.   By turning a model's own uncertainty into useful self-feedback, RLSF affirms reinforcement learning on intrinsic model behaviour as a principled and data-efficient component of the LLM post-training pipeline and warrents further research in intrinsic rewards for LLM post-training.",
      "arxiv_id": "2507.21931v1",
      "published": "2025-07-29",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2507.21931v1",
      "pdf_url": "https://arxiv.org/pdf/2507.21931v1.pdf"
    },
    {
      "title": "Activation Control for Efficiently Eliciting Long Chain-of-thought Ability of Language Models",
      "authors": [
        "Zekai Zhao",
        "Qi Liu",
        "Kun Zhou",
        "Zihan Liu",
        "Yifei Shao",
        "Zhiting Hu",
        "Biwei Huang"
      ],
      "summary": "Despite the remarkable reasoning performance, eliciting the long chain-of-thought (CoT) ability in large language models (LLMs) typically requires costly reinforcement learning or supervised fine-tuning on high-quality distilled data. We investigate the internal mechanisms behind this capability and show that a small set of high-impact activations in the last few layers largely governs long-form reasoning attributes, such as output length and self-reflection. By simply amplifying these activations and inserting \"wait\" tokens, we can invoke the long CoT ability without any training, resulting in significantly increased self-reflection rates and accuracy. Moreover, we find that the activation dynamics follow predictable trajectories, with a sharp rise after special tokens and a subsequent exponential decay. Building on these insights, we introduce a general training-free activation control technique. It leverages a few contrastive examples to identify key activations, and employs simple analytic functions to modulate their values at inference time to elicit long CoTs. Extensive experiments confirm the effectiveness of our method in efficiently eliciting long CoT reasoning in LLMs and improving their performance. Additionally, we propose a parameter-efficient fine-tuning method that trains only a last-layer activation amplification module and a few LoRA layers, outperforming full LoRA fine-tuning on reasoning benchmarks with significantly fewer parameters. Our code and data are publicly released.",
      "arxiv_id": "2505.17697v1",
      "published": "2025-05-23",
      "categories": [
        "cs.CL",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.17697v1",
      "pdf_url": "https://arxiv.org/pdf/2505.17697v1.pdf"
    },
    {
      "title": "SARI: Structured Audio Reasoning via Curriculum-Guided Reinforcement Learning",
      "authors": [
        "Cheng Wen",
        "Tingwei Guo",
        "Shuaijiang Zhao",
        "Wei Zou",
        "Xiangang Li"
      ],
      "summary": "Recent work shows that reinforcement learning(RL) can markedly sharpen the reasoning ability of large language models (LLMs) by prompting them to \"think before answering.\" Yet whether and how these gains transfer to audio-language reasoning remains largely unexplored. We extend the Group-Relative Policy Optimization (GRPO) framework from DeepSeek-R1 to a Large Audio-Language Model (LALM), and construct a 32k sample multiple-choice corpus. Using a two-stage regimen supervised fine-tuning on structured and unstructured chains-of-thought, followed by curriculum-guided GRPO, we systematically compare implicit vs. explicit, and structured vs. free form reasoning under identical architectures. Our structured audio reasoning model, SARI (Structured Audio Reasoning via Curriculum-Guided Reinforcement Learning), achieves a 16.35% improvement in average accuracy over the base model Qwen2-Audio-7B-Instruct. Furthermore, the variant built upon Qwen2.5-Omni reaches state-of-the-art performance of 67.08% on the MMAU test-mini benchmark. Ablation experiments show that on the base model we use: (i) SFT warm-up is important for stable RL training, (ii) structured chains yield more robust generalization than unstructured ones, and (iii) easy-to-hard curricula accelerate convergence and improve final performance. These findings demonstrate that explicit, structured reasoning and curriculum learning substantially enhances audio-language understanding.",
      "arxiv_id": "2504.15900v3",
      "published": "2025-04-22",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.15900v3",
      "pdf_url": "https://arxiv.org/pdf/2504.15900v3.pdf"
    },
    {
      "title": "Pixel Reasoner: Incentivizing Pixel-Space Reasoning with Curiosity-Driven Reinforcement Learning",
      "authors": [
        "Haozhe Wang",
        "Alex Su",
        "Weiming Ren",
        "Fangzhen Lin",
        "Wenhu Chen"
      ],
      "summary": "Chain-of-thought reasoning has significantly improved the performance of Large Language Models (LLMs) across various domains. However, this reasoning process has been confined exclusively to textual space, limiting its effectiveness in visually intensive tasks. To address this limitation, we introduce the concept of reasoning in the pixel-space. Within this novel framework, Vision-Language Models (VLMs) are equipped with a suite of visual reasoning operations, such as zoom-in and select-frame. These operations enable VLMs to directly inspect, interrogate, and infer from visual evidences, thereby enhancing reasoning fidelity for visual tasks. Cultivating such pixel-space reasoning capabilities in VLMs presents notable challenges, including the model's initially imbalanced competence and its reluctance to adopt the newly introduced pixel-space operations. We address these challenges through a two-phase training approach. The first phase employs instruction tuning on synthesized reasoning traces to familiarize the model with the novel visual operations. Following this, a reinforcement learning (RL) phase leverages a curiosity-driven reward scheme to balance exploration between pixel-space reasoning and textual reasoning. With these visual operations, VLMs can interact with complex visual inputs, such as information-rich images or videos to proactively gather necessary information. We demonstrate that this approach significantly improves VLM performance across diverse visual reasoning benchmarks. Our 7B model, \\model, achieves 84\\% on V* bench, 74\\% on TallyQA-Complex, and 84\\% on InfographicsVQA, marking the highest accuracy achieved by any open-source model to date. These results highlight the importance of pixel-space reasoning and the effectiveness of our framework.",
      "arxiv_id": "2505.15966v3",
      "published": "2025-05-21",
      "categories": [
        "cs.CV",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.15966v3",
      "pdf_url": "https://arxiv.org/pdf/2505.15966v3.pdf"
    },
    {
      "title": "RecLLM-R1: A Two-Stage Training Paradigm with Reinforcement Learning and Chain-of-Thought v1",
      "authors": [
        "Yu Xie",
        "Xingkai Ren",
        "Ying Qi",
        "Yao Hu",
        "Lianlei Shan"
      ],
      "summary": "Traditional recommendation systems often grapple with \"filter bubbles\", underutilization of external knowledge, and a disconnect between model optimization and business policy iteration. To address these limitations, this paper introduces RecLLM-R1, a novel recommendation framework leveraging Large Language Models (LLMs) and drawing inspiration from the DeepSeek R1 methodology. The framework initiates by transforming user profiles, historical interactions, and multi-faceted item attributes into LLM-interpretable natural language prompts through a carefully engineered data construction process. Subsequently, a two-stage training paradigm is employed: the initial stage involves Supervised Fine-Tuning (SFT) to imbue the LLM with fundamental recommendation capabilities. The subsequent stage utilizes Group Relative Policy Optimization (GRPO), a reinforcement learning technique, augmented with a Chain-of-Thought (CoT) mechanism. This stage guides the model through multi-step reasoning and holistic decision-making via a flexibly defined reward function, aiming to concurrently optimize recommendation accuracy, diversity, and other bespoke business objectives. Empirical evaluations on a real-world user behavior dataset from a large-scale social media platform demonstrate that RecLLM-R1 significantly surpasses existing baseline methods across a spectrum of evaluation metrics, including accuracy, diversity, and novelty. It effectively mitigates the filter bubble effect and presents a promising avenue for the integrated optimization of recommendation models and policies under intricate business goals.",
      "arxiv_id": "2506.19235v1",
      "published": "2025-06-24",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.19235v1",
      "pdf_url": "https://arxiv.org/pdf/2506.19235v1.pdf"
    },
    {
      "title": "MedVLM-R1: Incentivizing Medical Reasoning Capability of Vision-Language Models (VLMs) via Reinforcement Learning",
      "authors": [
        "Jiazhen Pan",
        "Che Liu",
        "Junde Wu",
        "Fenglin Liu",
        "Jiayuan Zhu",
        "Hongwei Bran Li",
        "Chen Chen",
        "Cheng Ouyang",
        "Daniel Rueckert"
      ],
      "summary": "Reasoning is a critical frontier for advancing medical image analysis, where transparency and trustworthiness play a central role in both clinician trust and regulatory approval. Although Medical Visual Language Models (VLMs) show promise for radiological tasks, most existing VLMs merely produce final answers without revealing the underlying reasoning. To address this gap, we introduce MedVLM-R1, a medical VLM that explicitly generates natural language reasoning to enhance transparency and trustworthiness. Instead of relying on supervised fine-tuning (SFT), which often suffers from overfitting to training distributions and fails to foster genuine reasoning, MedVLM-R1 employs a reinforcement learning framework that incentivizes the model to discover human-interpretable reasoning paths without using any reasoning references. Despite limited training data (600 visual question answering samples) and model parameters (2B), MedVLM-R1 boosts accuracy from 55.11% to 78.22% across MRI, CT, and X-ray benchmarks, outperforming larger models trained on over a million samples. It also demonstrates robust domain generalization under out-of-distribution tasks. By unifying medical image analysis with explicit reasoning, MedVLM-R1 marks a pivotal step toward trustworthy and interpretable AI in clinical practice. Inference model is available at: https://huggingface.co/JZPeterPan/MedVLM-R1.",
      "arxiv_id": "2502.19634v2",
      "published": "2025-02-26",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2502.19634v2",
      "pdf_url": "https://arxiv.org/pdf/2502.19634v2.pdf"
    },
    {
      "title": "Mental Modeling of Reinforcement Learning Agents by Language Models",
      "authors": [
        "Wenhao Lu",
        "Xufeng Zhao",
        "Josua Spisak",
        "Jae Hee Lee",
        "Stefan Wermter"
      ],
      "summary": "Can emergent language models faithfully model the intelligence of decision-making agents? Though modern language models exhibit already some reasoning ability, and theoretically can potentially express any probable distribution over tokens, it remains underexplored how the world knowledge these pretrained models have memorized can be utilized to comprehend an agent's behaviour in the physical world. This study empirically examines, for the first time, how well large language models (LLMs) can build a mental model of agents, termed agent mental modelling, by reasoning about an agent's behaviour and its effect on states from agent interaction history. This research may unveil the potential of leveraging LLMs for elucidating RL agent behaviour, addressing a key challenge in eXplainable reinforcement learning (XRL). To this end, we propose specific evaluation metrics and test them on selected RL task datasets of varying complexity, reporting findings on agent mental model establishment. Our results disclose that LLMs are not yet capable of fully mental modelling agents through inference alone without further innovations. This work thus provides new insights into the capabilities and limitations of modern LLMs.",
      "arxiv_id": "2406.18505v1",
      "published": "2024-06-26",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2406.18505v1",
      "pdf_url": "https://arxiv.org/pdf/2406.18505v1.pdf"
    },
    {
      "title": "Effective Reinforcement Learning for Reasoning in Language Models",
      "authors": [
        "Lianghuan Huang",
        "Shuo Li",
        "Sagnik Anupam",
        "Insup Lee",
        "Osbert Bastani"
      ],
      "summary": "Reinforcement learning (RL) has emerged as a promising strategy for improving the reasoning capabilities of language models (LMs) in domains such as mathematics and coding. However, most modern RL algorithms were designed to target robotics applications, which differ significantly from LM reasoning. We analyze RL algorithm design decisions for LM reasoning, for both accuracy and computational efficiency, focusing on relatively small models due to computational constraints. Our findings are: (i) on-policy RL significantly outperforms supervised fine-tuning (SFT), (ii) PPO-based off-policy updates increase accuracy instead of reduce variance, and (iii) removing KL divergence can lead to more concise generations and higher accuracy. Furthermore, we find that a key bottleneck to computational efficiency is that the optimal batch sizes for inference and backpropagation are different. We propose a novel algorithm, DASH, that performs preemptive sampling (i.e., sample a large batch and accumulate gradient updates in small increments), and gradient filtering (i.e., drop samples with small advantage estimates). We show that DASH reduces training time by 83% compared to a standard implementation of GRPO without sacrificing accuracy. Our findings provide valuable insights on designing effective RL algorithms for LM reasoning.",
      "arxiv_id": "2505.17218v1",
      "published": "2025-05-22",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.17218v1",
      "pdf_url": "https://arxiv.org/pdf/2505.17218v1.pdf"
    },
    {
      "title": "Knowledge Graph Reasoning with Self-supervised Reinforcement Learning",
      "authors": [
        "Ying Ma",
        "Owen Burns",
        "Mingqiu Wang",
        "Gang Li",
        "Nan Du",
        "Laurent El Shafey",
        "Liqiang Wang",
        "Izhak Shafran",
        "Hagen Soltau"
      ],
      "summary": "Reinforcement learning (RL) is an effective method of finding reasoning pathways in incomplete knowledge graphs (KGs). To overcome the challenges of a large action space, a self-supervised pre-training method is proposed to warm up the policy network before the RL training stage. To alleviate the distributional mismatch issue in general self-supervised RL (SSRL), in our supervised learning (SL) stage, the agent selects actions based on the policy network and learns from generated labels; this self-generation of labels is the intuition behind the name self-supervised. With this training framework, the information density of our SL objective is increased and the agent is prevented from getting stuck with the early rewarded paths. Our self-supervised RL (SSRL) method improves the performance of RL by pairing it with the wide coverage achieved by SL during pretraining, since the breadth of the SL objective makes it infeasible to train an agent with that alone. We show that our SSRL model meets or exceeds current state-of-the-art results on all Hits@k and mean reciprocal rank (MRR) metrics on four large benchmark KG datasets. This SSRL method can be used as a plug-in for any RL architecture for a KGR task. We adopt two RL architectures, i.e., MINERVA and MultiHopKG as our baseline RL models and experimentally show that our SSRL model consistently outperforms both baselines on all of these four KG reasoning tasks. Full code for the paper available at https://github.com/owenonline/Knowledge-Graph-Reasoning-with-Self-supervised-Reinforcement-Learning.",
      "arxiv_id": "2405.13640v2",
      "published": "2024-05-22",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2405.13640v2",
      "pdf_url": "https://arxiv.org/pdf/2405.13640v2.pdf"
    },
    {
      "title": "Audio-Thinker: Guiding Audio Language Model When and How to Think via Reinforcement Learning",
      "authors": [
        "Shu Wu",
        "Chenxing Li",
        "Wenfu Wang",
        "Hao Zhang",
        "Hualei Wang",
        "Meng Yu",
        "Dong Yu"
      ],
      "summary": "Recent advancements in large language models, multimodal large language models, and large audio language models (LALMs) have significantly improved their reasoning capabilities through reinforcement learning with rule-based rewards. However, the explicit reasoning process has yet to show significant benefits for audio question answering, and effectively leveraging deep reasoning remains an open challenge, with LALMs still falling short of human-level auditory-language reasoning. To address these limitations, we propose Audio-Thinker, a reinforcement learning framework designed to enhance the reasoning capabilities of LALMs, with a focus on improving adaptability, consistency, and effectiveness. Our approach introduces an adaptive think accuracy reward, enabling the model to adjust its reasoning strategies based on task complexity dynamically. Furthermore, we incorporate an external reward model to evaluate the overall consistency and quality of the reasoning process, complemented by think-based rewards that help the model distinguish between valid and flawed reasoning paths during training. Experimental results demonstrate that our Audio-Thinker model outperforms existing reasoning-oriented LALMs across various benchmark tasks, exhibiting superior reasoning and generalization capabilities.",
      "arxiv_id": "2508.08039v3",
      "published": "2025-08-11",
      "categories": [
        "cs.SD",
        "cs.CL",
        "cs.MM",
        "eess.AS"
      ],
      "url": "https://arxiv.org/abs/2508.08039v3",
      "pdf_url": "https://arxiv.org/pdf/2508.08039v3.pdf"
    },
    {
      "title": "RedStar: Does Scaling Long-CoT Data Unlock Better Slow-Reasoning Systems?",
      "authors": [
        "Haotian Xu",
        "Xing Wu",
        "Weinong Wang",
        "Zhongzhi Li",
        "Da Zheng",
        "Boyuan Chen",
        "Yi Hu",
        "Shijia Kang",
        "Jiaming Ji",
        "Yingying Zhang",
        "Zhijiang Guo",
        "Yaodong Yang",
        "Muhan Zhang",
        "Debing Zhang"
      ],
      "summary": "Can scaling transform reasoning? In this work, we explore the untapped potential of scaling Long Chain-of-Thought (Long-CoT) data to 1000k samples, pioneering the development of a slow-thinking model, RedStar. Through extensive experiments with various LLMs and different sizes, we uncover the ingredients for specialization and scale for Long-CoT training. Surprisingly, even smaller models show significant performance gains with limited data, revealing the sample efficiency of Long-CoT and the critical role of sample difficulty in the learning process. Our findings demonstrate that Long-CoT reasoning can be effectively triggered with just a few thousand examples, while larger models achieve unparalleled improvements. We also introduce reinforcement learning (RL)-scale training as a promising direction for advancing slow-thinking systems. RedStar shines across domains: on the MATH-Hard benchmark, RedStar-code-math boosts performance from 66.2\\% to 81.6\\%, and on the USA Math Olympiad (AIME), it solves 46.7\\% of problems using only 21k mixed-code-math datasets. In multimodal tasks like GeoQA and MathVista-GEO, RedStar-Geo achieves competitive results with minimal Long-CoT data, outperforming other slow-thinking systems like QvQ-Preview. Compared to QwQ, RedStar strikes the perfect balance between reasoning and generalizability. Our work highlights that, with careful tuning, scaling Long-CoT can unlock extraordinary reasoning capabilities-even with limited dataset and set a new standard for slow-thinking models across diverse challenges. Our data and models are released at https://huggingface.co/RedStar-Reasoning.",
      "arxiv_id": "2501.11284v1",
      "published": "2025-01-20",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2501.11284v1",
      "pdf_url": "https://arxiv.org/pdf/2501.11284v1.pdf"
    },
    {
      "title": "ReaLM: Reflection-Enhanced Autonomous Reasoning with Small Language Models",
      "authors": [
        "Yuanfeng Xu",
        "Zehui Dai",
        "Jian Liang",
        "Jiapeng Guan",
        "Guangrun Wang",
        "Liang Lin",
        "Xiaohui Lv"
      ],
      "summary": "Small Language Models (SLMs) are a cost-effective alternative to Large Language Models (LLMs), but often struggle with complex reasoning due to their limited capacity and a tendency to produce mistakes or inconsistent answers during multi-step reasoning. Existing efforts have improved SLM performance, but typically at the cost of one or more of three key aspects: (1) reasoning capability, due to biased supervision that filters out negative reasoning paths and limits learning from errors; (2) autonomy, due to over-reliance on externally generated reasoning signals; and (3) generalization, which suffers when models overfit to teacher-specific patterns. In this paper, we introduce ReaLM, a reinforcement learning framework for robust and self-sufficient reasoning in vertical domains. To enhance reasoning capability, we propose Multi-Route Process Verification (MRPV), which contrasts both positive and negative reasoning paths to extract decisive patterns. To reduce reliance on external guidance and improve autonomy, we introduce Enabling Autonomy via Asymptotic Induction (EAAI), a training strategy that gradually fades external signals. To improve generalization, we apply guided chain-of-thought distillation to encode domain-specific rules and expert knowledge into SLM parameters, making them part of what the model has learned. Extensive experiments on both vertical and general reasoning tasks demonstrate that ReaLM significantly improves SLM performance across aspects (1)-(3) above.",
      "arxiv_id": "2508.12387v1",
      "published": "2025-08-17",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2508.12387v1",
      "pdf_url": "https://arxiv.org/pdf/2508.12387v1.pdf"
    },
    {
      "title": "Reasoning-Table: Exploring Reinforcement Learning for Table Reasoning",
      "authors": [
        "Fangyu Lei",
        "Jinxiang Meng",
        "Yiming Huang",
        "Tinghong Chen",
        "Yun Zhang",
        "Shizhu He",
        "Jun Zhao",
        "Kang Liu"
      ],
      "summary": "Table reasoning, encompassing tasks such as table question answering, fact verification, and text-to-SQL, requires precise understanding of structured tabular data, coupled with numerical computation and code manipulation for effective inference. Supervised fine-tuning (SFT) approaches have achieved notable success but often struggle with generalization and robustness due to biases inherent in imitative learning. We introduce Reasoning-Table, the first application of reinforcement learning (RL) to table reasoning, achieving state-of-the-art performance. Through rigorous data preprocessing, reward design, and tailored training strategies, our method leverages simple rule-based outcome rewards to outperform SFT across multiple benchmarks. Unified training across diverse tasks enables Reasoning-Table to emerge as a robust table reasoning large language model, surpassing larger proprietary models like Claude-3.7-Sonnet by 4.0% on table reasoning benchmarks. The approach also achieves excellent performance on text-to-SQL tasks, reaching 68.3% performance on the BIRD dev dataset with a 7B model. Further experiments demonstrate that Reasoning-Table enhances the model's generalization capabilities and robustness.",
      "arxiv_id": "2506.01710v1",
      "published": "2025-06-02",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.01710v1",
      "pdf_url": "https://arxiv.org/pdf/2506.01710v1.pdf"
    },
    {
      "title": "LoongRL: Reinforcement Learning for Advanced Reasoning over Long Contexts",
      "authors": [
        "Siyuan Wang",
        "Gaokai Zhang",
        "Li Lyna Zhang",
        "Ning Shang",
        "Fan Yang",
        "Dongyao Chen",
        "Mao Yang"
      ],
      "summary": "Reasoning over long contexts is essential for large language models. While reinforcement learning (RL) enhances short-context reasoning by inducing \"Aha\" moments in chain-of-thought, the advanced thinking patterns required for long-context reasoning remain largely unexplored, and high-difficulty RL data are scarce. In this paper, we introduce LoongRL, a data-driven RL method for advanced long-context reasoning. Central to LoongRL is KeyChain, a synthesis approach that transforms short multi-hop QA into high-difficulty long-context tasks by inserting UUID chains that hide the true question among large collections of distracting documents. Solving these tasks requires the model to trace the correct chain step-by-step, identify the true question, retrieve relevant facts and reason over them to answer correctly. RL training on KeyChain data induces an emergent plan-retrieve-reason-recheck reasoning pattern that generalizes far beyond training length. Models trained at 16K effectively solve 128K tasks without prohibitive full-length RL rollout costs. On Qwen2.5-7B and 14B, LoongRL substantially improves long-context multi-hop QA accuracy by +23.5% and +21.1% absolute gains. The resulting LoongRL-14B reaches a score of 74.2, rivaling much larger frontier models such as o3-mini (74.5) and DeepSeek-R1 (74.9). It also improves long-context retrieval, passes all 128K needle-in-a-haystack stress tests, and preserves short-context reasoning capabilities.",
      "arxiv_id": "2510.19363v2",
      "published": "2025-10-22",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.19363v2",
      "pdf_url": "https://arxiv.org/pdf/2510.19363v2.pdf"
    },
    {
      "title": "The Ends Justify the Thoughts: RL-Induced Motivated Reasoning in LLMs",
      "authors": [
        "Nikolaus Howe",
        "Micah Carroll"
      ],
      "summary": "The use of reinforcement learning (RL) with chain-of-thought (CoT) reasoning has emerged as a promising approach for developing more capable language models. In turn, this has led to investigation of CoT monitoring as a compelling method for detecting harmful behaviors such as reward hacking, under the assumption that models' reasoning processes reflect their internal decision-making. In practice, LLM training often produces unintended behaviors due to imperfect reward signals, leading models to develop misaligned tendencies. A common corrective approach is to apply post-hoc instructions to avoid problematic behaviors like sycophancy, but what happens to the model's reasoning process when these instructions conflict with learned behaviors? We investigate this question in simple settings and find that models engage in systematic motivated reasoning -- generating plausible-sounding justifications for violating their instructions while downplaying potential harms. Beyond being an interesting property of training, we find that while motivated reasoning can be detected by most frontier reasoning models, smaller LLM judges can fail to identify a portion of it, and in rare cases can themselves be persuaded that the reasoning is correct, despite it contradicting clear instructions. This capability gap raises concerns that as models become more sophisticated, their motivated reasoning may become increasingly difficult for monitors to detect. Our results underscore the need to account for motivated reasoning when relying on chain-of-thought processes for model evaluation and oversight. All code for this paper will be made available. WARNING: some examples in this paper may be upsetting.",
      "arxiv_id": "2510.17057v1",
      "published": "2025-10-20",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2510.17057v1",
      "pdf_url": "https://arxiv.org/pdf/2510.17057v1.pdf"
    },
    {
      "title": "PRISM: Preference Refinement via Implicit Scene Modeling for 3D Vision-Language Preference-Based Reinforcement Learning",
      "authors": [
        "Yirong Sun",
        "Yanjun Chen"
      ],
      "summary": "We propose PRISM, a novel framework designed to overcome the limitations of 2D-based Preference-Based Reinforcement Learning (PBRL) by unifying 3D point cloud modeling and future-aware preference refinement. At its core, PRISM adopts a 3D Point Cloud-Language Model (3D-PC-LLM) to mitigate occlusion and viewpoint biases, ensuring more stable and spatially consistent preference signals. Additionally, PRISM leverages Chain-of-Thought (CoT) reasoning to incorporate long-horizon considerations, thereby preventing the short-sighted feedback often seen in static preference comparisons. In contrast to conventional PBRL techniques, this integration of 3D perception and future-oriented reasoning leads to significant gains in preference agreement rates, faster policy convergence, and robust generalization across unseen robotic environments. Our empirical results, spanning tasks such as robotic manipulation and autonomous navigation, highlight PRISM's potential for real-world applications where precise spatial understanding and reliable long-term decision-making are critical. By bridging 3D geometric awareness with CoT-driven preference modeling, PRISM establishes a comprehensive foundation for scalable, human-aligned reinforcement learning.",
      "arxiv_id": "2503.10177v2",
      "published": "2025-03-13",
      "categories": [
        "cs.CL",
        "cs.RO"
      ],
      "url": "https://arxiv.org/abs/2503.10177v2",
      "pdf_url": "https://arxiv.org/pdf/2503.10177v2.pdf"
    },
    {
      "title": "AttriLens-Mol: Attribute Guided Reinforcement Learning for Molecular Property Prediction with Large Language Models",
      "authors": [
        "Xuan Lin",
        "Long Chen",
        "Yile Wang"
      ],
      "summary": "Large Language Models (LLMs) have shown promise in assisting molecular property prediction tasks but often rely on human-crafted prompts and chain-of-thought templates. While recent advanced large reasoning models like DeepSeek-R1 employ reinforcement learning for an extended ``thinking'' process, their reasoning can be verbose and lack relevance. We introduce AttriLens-Mol, an attribute-guided reinforcement learning framework for molecular property prediction with LLMs. AttriLens-Mol steers the model's reasoning by using: (1) a format reward encouraging attribute-based structured output, (2) a count reward to avoid enumerating irrelevant attributes, and (3) a rationality reward using advanced LLMs and RDKit to verify the relatedness of the generated attributes. This approach implicitly elicits the model's inherent knowledge of relevant molecular attributes during reasoning, enables making predictions for the molecular property more effectively. Experiments on both in-distribution and out-of-distribution datasets show that, training both 7B-size R1-Distilled-Qwen2.5 and R1-Distilled-LLaMA3.1 models on 4,000 samples with our proposed AttriLens-Mol method significantly boosts the performance, getting comparable or better results than supervised fine-tuning models (Mol-Instructions, ChemDFM, etc.) and advanced models (GPT-3.5, GPT-4o, DeepSeek-V3, DeepSeek-R1, etc.). Further, our extracted attributes for the target property, when used as features for an interpretable decision tree model, yield superior performance compared to attributes generated by prompting LLMs. This shows that AttriLens-Mol effectively elicits more relevant and predictive molecular attributes, leading to enhanced interpretability and performance for property prediction. We release the code in https://github.com/szu-tera/AttriLens-Mol.",
      "arxiv_id": "2508.04748v2",
      "published": "2025-08-06",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2508.04748v2",
      "pdf_url": "https://arxiv.org/pdf/2508.04748v2.pdf"
    },
    {
      "title": "Towards Effective Code-Integrated Reasoning",
      "authors": [
        "Fei Bai",
        "Yingqian Min",
        "Beichen Zhang",
        "Zhipeng Chen",
        "Wayne Xin Zhao",
        "Lei Fang",
        "Zheng Liu",
        "Zhongyuan Wang",
        "Ji-Rong Wen"
      ],
      "summary": "In this paper, we investigate code-integrated reasoning, where models generate code when necessary and integrate feedback by executing it through a code interpreter. To acquire this capability, models must learn when and how to use external code tools effectively, which is supported by tool-augmented reinforcement learning (RL) through interactive learning. Despite its benefits, tool-augmented RL can still suffer from potential instability in the learning dynamics. In light of this challenge, we present a systematic approach to improving the training effectiveness and stability of tool-augmented RL for code-integrated reasoning. Specifically, we develop enhanced training strategies that balance exploration and stability, progressively building tool-use capabilities while improving reasoning performance. Through extensive experiments on five mainstream mathematical reasoning benchmarks, our model demonstrates significant performance improvements over multiple competitive baselines. Furthermore, we conduct an in-depth analysis of the mechanism and effect of code-integrated reasoning, revealing several key insights, such as the extension of model's capability boundaries and the simultaneous improvement of reasoning efficiency through code integration. All data and code for reproducing this work are available at: https://github.com/RUCAIBox/CIR.",
      "arxiv_id": "2505.24480v1",
      "published": "2025-05-30",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.24480v1",
      "pdf_url": "https://arxiv.org/pdf/2505.24480v1.pdf"
    },
    {
      "title": "Reason-RFT: Reinforcement Fine-Tuning for Visual Reasoning of Vision Language Models",
      "authors": [
        "Huajie Tan",
        "Yuheng Ji",
        "Xiaoshuai Hao",
        "Xiansheng Chen",
        "Pengwei Wang",
        "Zhongyuan Wang",
        "Shanghang Zhang"
      ],
      "summary": "Visual reasoning abilities play a crucial role in understanding complex multimodal data, advancing both domain-specific applications and artificial general intelligence (AGI). Existing methods enhance Vision-Language Models (VLMs) through Chain-of-Thought (CoT) supervised fine-tuning using meticulously annotated data. However, this approach may lead to overfitting and cognitive rigidity, limiting the model's generalization ability under domain shifts and reducing real-world applicability. To overcome these limitations, we propose Reason-RFT, a two-stage reinforcement fine-tuning framework for visual reasoning. First, Supervised Fine-Tuning (SFT) with curated CoT data activates the reasoning potential of VLMs. This is followed by reinforcement learning based on Group Relative Policy Optimization (GRPO), which generates multiple reasoning-response pairs to enhance adaptability to domain shifts. To evaluate Reason-RFT, we reconstructed a comprehensive dataset covering visual counting, structural perception, and spatial transformation, serving as a benchmark for systematic assessment across three key dimensions. Experimental results highlight three advantages: (1) performance enhancement, with Reason-RFT achieving state-of-the-art results and outperforming both open-source and proprietary models; (2) generalization superiority, maintaining robust performance under domain shifts across various tasks; and (3) data efficiency, excelling in few-shot learning scenarios and surpassing full-dataset SFT baselines. Reason-RFT introduces a novel training paradigm for visual reasoning and marks a significant step forward in multimodal research. Project website: https://tanhuajie.github.io/ReasonRFT",
      "arxiv_id": "2503.20752v3",
      "published": "2025-03-26",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.20752v3",
      "pdf_url": "https://arxiv.org/pdf/2503.20752v3.pdf"
    },
    {
      "title": "A Survey of Reinforcement Learning for Large Reasoning Models",
      "authors": [
        "Kaiyan Zhang",
        "Yuxin Zuo",
        "Bingxiang He",
        "Youbang Sun",
        "Runze Liu",
        "Che Jiang",
        "Yuchen Fan",
        "Kai Tian",
        "Guoli Jia",
        "Pengfei Li",
        "Yu Fu",
        "Xingtai Lv",
        "Yuchen Zhang",
        "Sihang Zeng",
        "Shang Qu",
        "Haozhan Li",
        "Shijie Wang",
        "Yuru Wang",
        "Xinwei Long",
        "Fangfu Liu",
        "Xiang Xu",
        "Jiaze Ma",
        "Xuekai Zhu",
        "Ermo Hua",
        "Yihao Liu",
        "Zonglin Li",
        "Huayu Chen",
        "Xiaoye Qu",
        "Yafu Li",
        "Weize Chen",
        "Zhenzhao Yuan",
        "Junqi Gao",
        "Dong Li",
        "Zhiyuan Ma",
        "Ganqu Cui",
        "Zhiyuan Liu",
        "Biqing Qi",
        "Ning Ding",
        "Bowen Zhou"
      ],
      "summary": "In this paper, we survey recent advances in Reinforcement Learning (RL) for reasoning with Large Language Models (LLMs). RL has achieved remarkable success in advancing the frontier of LLM capabilities, particularly in addressing complex logical tasks such as mathematics and coding. As a result, RL has emerged as a foundational methodology for transforming LLMs into LRMs. With the rapid progress of the field, further scaling of RL for LRMs now faces foundational challenges not only in computational resources but also in algorithm design, training data, and infrastructure. To this end, it is timely to revisit the development of this domain, reassess its trajectory, and explore strategies to enhance the scalability of RL toward Artificial SuperIntelligence (ASI). In particular, we examine research applying RL to LLMs and LRMs for reasoning abilities, especially since the release of DeepSeek-R1, including foundational components, core problems, training resources, and downstream applications, to identify future opportunities and directions for this rapidly evolving area. We hope this review will promote future research on RL for broader reasoning models. Github: https://github.com/TsinghuaC3I/Awesome-RL-for-LRMs",
      "arxiv_id": "2509.08827v3",
      "published": "2025-09-10",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2509.08827v3",
      "pdf_url": "https://arxiv.org/pdf/2509.08827v3.pdf"
    },
    {
      "title": "Semi-off-Policy Reinforcement Learning for Vision-Language Slow-Thinking Reasoning",
      "authors": [
        "Junhao Shen",
        "Haiteng Zhao",
        "Yuzhe Gu",
        "Songyang Gao",
        "Kuikun Liu",
        "Haian Huang",
        "Jianfei Gao",
        "Dahua Lin",
        "Wenwei Zhang",
        "Kai Chen"
      ],
      "summary": "Enhancing large vision-language models (LVLMs) with visual slow-thinking reasoning is crucial for solving complex multimodal tasks. However, since LVLMs are mainly trained with vision-language alignment, it is difficult to adopt on-policy reinforcement learning (RL) to develop the slow thinking ability because the rollout space is restricted by its initial abilities. Off-policy RL offers a way to go beyond the current policy, but directly distilling trajectories from external models may cause visual hallucinations due to mismatched visual perception abilities across models. To address these issues, this paper proposes SOPHIA, a simple and scalable Semi-Off-Policy RL for vision-language slow-tHInking reAsoning. SOPHIA builds a semi-off-policy behavior model by combining on-policy visual understanding from a trainable LVLM with off-policy slow-thinking reasoning from a language model, assigns outcome-based rewards to reasoning, and propagates visual rewards backward. Then LVLM learns slow-thinking reasoning ability from the obtained reasoning trajectories using propagated rewards via off-policy RL algorithms. Extensive experiments with InternVL2.5 and InternVL3.0 with 8B and 38B sizes show the effectiveness of SOPHIA. Notably, SOPHIA improves InternVL3.0-38B by 8.50% in average, reaching state-of-the-art performance among open-source LVLMs on multiple multimodal reasoning benchmarks, and even outperforms some closed-source models (e.g., GPT-4.1) on the challenging MathVision and OlympiadBench, achieving 49.08% and 49.95% pass@1 accuracy, respectively. Analysis shows SOPHIA outperforms supervised fine-tuning and direct on-policy RL methods, offering a better policy initialization for further on-policy training.",
      "arxiv_id": "2507.16814v2",
      "published": "2025-07-22",
      "categories": [
        "cs.LG",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2507.16814v2",
      "pdf_url": "https://arxiv.org/pdf/2507.16814v2.pdf"
    },
    {
      "title": "Geo-R1: Unlocking VLM Geospatial Reasoning with Cross-View Reinforcement Learning",
      "authors": [
        "Chenhui Xu",
        "Fuxun Yu",
        "Michael J. Bianco",
        "Jacob Kovarskiy",
        "Raphael Tang",
        "Qi Zhang",
        "Zirui Xu",
        "Will LeVine",
        "Brandon Dubbs",
        "Heming Liao",
        "Cassandra Burgess",
        "Suvam Bag",
        "Jay Patravali",
        "Rupanjali Kukal",
        "Mikael Figueroa",
        "Rishi Madhok",
        "Nikolaos Karianakis",
        "Jinjun Xiong"
      ],
      "summary": "We introduce Geo-R1, a reasoning-centric post-training framework that unlocks geospatial reasoning in vision-language models by combining thinking scaffolding and elevating. In the scaffolding stage, Geo-R1 instills a ``geospatial thinking paradigm\" via supervised fine-tuning on synthetic chain-of-thought exemplars, enabling models to connect visual cues with geographic priors without costly human reasoning annotations. In the elevating stage, it uses GRPO-based reinforcement learning on a weakly-supervised cross-view pairing proxy. This design supplies a verifiable and scalable reward signal: teaching models to capture and reconcile features across modalities, and harnessing reasoning for accurate prediction. Geo-R1 extends geospatial modeling from domain pretraining / supervised finetuning to reasoning-first post-training, and achieves state-of-the-art performance across various geospatial reasoning benchmarks. Our model is available at https://huggingface.co/miniHui/Geo-R1.",
      "arxiv_id": "2510.00072v1",
      "published": "2025-09-29",
      "categories": [
        "cs.CV",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.00072v1",
      "pdf_url": "https://arxiv.org/pdf/2510.00072v1.pdf"
    },
    {
      "title": "Revolutionizing Reinforcement Learning Framework for Diffusion Large Language Models",
      "authors": [
        "Yinjie Wang",
        "Ling Yang",
        "Bowen Li",
        "Ye Tian",
        "Ke Shen",
        "Mengdi Wang"
      ],
      "summary": "We propose TraceRL, a trajectory-aware reinforcement learning framework for diffusion language models (DLMs) that incorporates preferred inference trajectory into post-training, and is applicable across different architectures. Equipped with a diffusion-based value model that enhances training stability, we demonstrate improved reasoning performance on complex math and coding tasks. Besides, it can also be applied to adapt block-specific models to larger blocks, which improves sampling flexibility. Employing TraceRL, we derive a series of state-of-the-art diffusion language models, namely TraDo. Although smaller than 7B-scale AR models, TraDo-4B-Instruct still consistently outperforms them across complex math reasoning tasks. TraDo-8B-Instruct achieves relative accuracy improvements of 6.1% over Qwen2.5-7B-Instruct and 51.3% over Llama3.1-8B-Instruct on mathematical reasoning benchmarks. Through curriculum learning, we also derive the first long-CoT DLM, outperforming Qwen2.5-7B-Instruct on MATH500 with an 18.1% relative accuracy gain. To facilitate reproducible research and practical applications, we release a comprehensive open-source framework for building, training, and deploying diffusion LLMs across diverse architectures. The framework integrates accelerated KV-cache techniques and inference engines for both inference and reinforcement learning, and includes implementations of various supervised fine-tuning and RL methods for mathematics, coding, and general tasks. Code and Models: https://github.com/Gen-Verse/dLLM-RL",
      "arxiv_id": "2509.06949v1",
      "published": "2025-09-08",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.06949v1",
      "pdf_url": "https://arxiv.org/pdf/2509.06949v1.pdf"
    },
    {
      "title": "Sparks of Tabular Reasoning via Text2SQL Reinforcement Learning",
      "authors": [
        "Josefa Lia Stoisser",
        "Marc Boubnovski Martell",
        "Julien Fauqueur"
      ],
      "summary": "This work reframes the Text-to-SQL task as a pathway for teaching large language models (LLMs) to reason over and manipulate tabular data--moving beyond the traditional focus on query generation. We propose a two-stage framework that leverages SQL supervision to develop transferable table reasoning capabilities. First, we synthesize detailed chain-of-thought (CoT) traces from real-world SQL queries, providing step-by-step, clause-level supervision that teaches the model how to traverse, filter, and aggregate table fields. Second, we introduce a Group Relative Policy Optimization (GRPO) reinforcement learning objective that connects SQL execution accuracy to generalizable reasoning by encouraging steps that extend beyond task-specific syntax and transfer across datasets. Empirically, our approach improves performance on standard Text-to-SQL benchmarks and achieves substantial gains on reasoning-intensive datasets such as BIRD and CRT-QA, demonstrating enhanced generalization and interpretability. Specifically, the distilled-quantized LLaMA model achieved a relative 33.9\\% increase in accuracy when trained on Text-to-SQL tasks, while Qwen achieved a relative 14.5\\% increase. These results suggest that SQL can serve not only as a target formalism but also as an effective scaffold for learning robust, transferable reasoning over structured data.",
      "arxiv_id": "2505.00016v2",
      "published": "2025-04-23",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.00016v2",
      "pdf_url": "https://arxiv.org/pdf/2505.00016v2.pdf"
    },
    {
      "title": "From Roots to Rewards: Dynamic Tree Reasoning with Reinforcement Learning",
      "authors": [
        "Ahmed Bahloul",
        "Simon Malberg"
      ],
      "summary": "Modern language models address complex questions through chain-of-thought (CoT) reasoning (Wei et al., 2023) and retrieval augmentation (Lewis et al., 2021), yet struggle with error propagation and knowledge integration. Tree-structured reasoning methods, particularly the Probabilistic Tree-of-Thought (ProbTree)(Cao et al., 2023) framework, mitigate these issues by decomposing questions into hierarchical structures and selecting answers through confidence-weighted aggregation of parametric and retrieved knowledge (Yao et al., 2023). However, ProbTree's static implementation introduces two key limitations: (1) the reasoning tree is fixed during the initial construction phase, preventing dynamic adaptation to intermediate results, and (2) each node requires exhaustive evaluation of all possible solution strategies, creating computational inefficiency. We present a dynamic reinforcement learning (Sutton and Barto, 2018) framework that transforms tree-based reasoning into an adaptive process. Our approach incrementally constructs the reasoning tree based on real-time confidence estimates, while learning optimal policies for action selection (decomposition, retrieval, or aggregation). This maintains ProbTree's probabilistic rigor while improving both solution quality and computational efficiency through selective expansion and focused resource allocation. The work establishes a new paradigm for treestructured reasoning that balances the reliability of probabilistic frameworks with the flexibility required for real-world question answering systems. Code available at: https://github.com/ahmedehabb/From-Roots-to-Rewards-Dynamic-Tree-Reasoning-with-RL",
      "arxiv_id": "2507.13142v4",
      "published": "2025-07-17",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2507.13142v4",
      "pdf_url": "https://arxiv.org/pdf/2507.13142v4.pdf"
    },
    {
      "title": "Conditional Advantage Estimation for Reinforcement Learning in Large Reasoning Models",
      "authors": [
        "Guanxu Chen",
        "Yafu Li",
        "Yuxian Jiang",
        "Chen Qian",
        "Qihan Ren",
        "Jingyi Yang",
        "Yu Cheng",
        "Dongrui Liu",
        "Jing Shao"
      ],
      "summary": "Reinforcement Learning with Verifiable Rewards (RLVR) for large language models (LLMs) has achieved remarkable progress in enhancing LLMs' reasoning capabilities on tasks with clear correctness criteria, such as mathematical reasoning tasks. Several training metrics, such as entropy or response length, have been observed to correlate with different reasoning behaviors in reinforcement learning. Prior approaches incorporate such priors through reward or advantage shaping, which often relies on hand-crafted penalties and preferences (e.g., higher-is-better or lower-is-better). However, without careful hyperparameter tuning, these directional priors can be overly biased and may lead to failure. To this end, we introduce Conditional advANtage estimatiON (CANON), amplifying the impact of the target metric without presuming its direction. Specifically, CANON regroups the sampled responses into two groups based on the higher or lower value of a target metric, measures which metric trend contributes to better performance through inter-group comparison, and identifies the better response within the same group. In summary, CANON based on entropy consistently outperforms prior methods across three LLMs on both math reasoning and high-complexity logic tasks. When applied to response length, CANON further improves token efficiency, yielding a more favorable Pareto frontier in the performance-cost trade-off.",
      "arxiv_id": "2509.23962v1",
      "published": "2025-09-28",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2509.23962v1",
      "pdf_url": "https://arxiv.org/pdf/2509.23962v1.pdf"
    },
    {
      "title": "Concise Reasoning via Reinforcement Learning",
      "authors": [
        "Mehdi Fatemi",
        "Banafsheh Rafiee",
        "Mingjie Tang",
        "Kartik Talamadupula"
      ],
      "summary": "Despite significant advancements in large language models (LLMs), a major drawback of reasoning models is their enormous token usage, which increases computational cost, resource requirements, and response time. In this work, we revisit the core principles of reinforcement learning (RL) and, through mathematical analysis, demonstrate that the tendency to generate lengthy responses arises inherently from RL-based optimization during training. This finding questions the prevailing assumption that longer responses inherently improve reasoning accuracy. Instead, we uncover a natural correlation between conciseness and accuracy that has been largely overlooked. We show that introducing a secondary phase of RL training, using a very small set of problems, can significantly reduce chains of thought while maintaining or even enhancing accuracy. Additionally, we demonstrate that, while GRPO shares some interesting properties of PPO, it suffers from collapse modes, which limit its reliability for concise reasoning. Finally, we validate our conclusions through extensive experimental results.",
      "arxiv_id": "2504.05185v2",
      "published": "2025-04-07",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.05185v2",
      "pdf_url": "https://arxiv.org/pdf/2504.05185v2.pdf"
    },
    {
      "title": "Improving Language Model Reasoning with Self-motivated Learning",
      "authors": [
        "Yunlong Feng",
        "Yang Xu",
        "Libo Qin",
        "Yasheng Wang",
        "Wanxiang Che"
      ],
      "summary": "Large-scale high-quality training data is important for improving the performance of models. After trained with data that has rationales (reasoning steps), models gain reasoning capability. However, the dataset with high-quality rationales is relatively scarce due to the high annotation cost. To address this issue, we propose \\textit{Self-motivated Learning} framework. The framework motivates the model itself to automatically generate rationales on existing datasets. Based on the inherent rank from correctness across multiple rationales, the model learns to generate better rationales, leading to higher reasoning capability. Specifically, we train a reward model with the rank to evaluate the quality of rationales, and improve the performance of reasoning through reinforcement learning. Experiment results of Llama2 7B on multiple reasoning datasets show that our method significantly improves the reasoning ability of models, even outperforming text-davinci-002 in some datasets.",
      "arxiv_id": "2404.07017v3",
      "published": "2024-04-10",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2404.07017v3",
      "pdf_url": "https://arxiv.org/pdf/2404.07017v3.pdf"
    },
    {
      "title": "Reinforce LLM Reasoning through Multi-Agent Reflection",
      "authors": [
        "Yurun Yuan",
        "Tengyang Xie"
      ],
      "summary": "Leveraging more test-time computation has proven to be an effective way to boost the reasoning capabilities of large language models (LLMs). Among various methods, the verify-and-improve paradigm stands out for enabling dynamic solution exploration and feedback incorporation. However, existing approaches often suffer from restricted feedback spaces and lack of coordinated training of different parties, leading to suboptimal performance. To address this, we model this multi-turn refinement process as a Markov Decision Process and introduce DPSDP (Direct Policy Search by Dynamic Programming), a reinforcement learning algorithm that trains an actor-critic LLM system to iteratively refine answers via direct preference learning on self-generated data. Theoretically, DPSDP can match the performance of any policy within the training distribution. Empirically, we instantiate DPSDP with various base models and show improvements on both in- and out-of-distribution benchmarks. For example, on benchmark MATH 500, majority voting over five refinement steps increases first-turn accuracy from 58.2% to 63.2% with Ministral-based models. An ablation study further confirms the benefits of multi-agent collaboration and out-of-distribution generalization.",
      "arxiv_id": "2506.08379v1",
      "published": "2025-06-10",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.08379v1",
      "pdf_url": "https://arxiv.org/pdf/2506.08379v1.pdf"
    },
    {
      "title": "Applications of Large Language Model Reasoning in Feature Generation",
      "authors": [
        "Dharani Chandra"
      ],
      "summary": "Large Language Models (LLMs) have revolutionized natural language processing through their state of art reasoning capabilities. This paper explores the convergence of LLM reasoning techniques and feature generation for machine learning tasks. We examine four key reasoning approaches: Chain of Thought, Tree of Thoughts, Retrieval-Augmented Generation, and Thought Space Exploration. Our analysis reveals how these approaches can be used to identify effective feature generation rules without having to manually specify search spaces. The paper categorizes LLM-based feature generation methods across various domains including finance, healthcare, and text analytics. LLMs can extract key information from clinical notes and radiology reports in healthcare, by enabling more efficient data utilization. In finance, LLMs facilitate text generation, summarization, and entity extraction from complex documents. We analyze evaluation methodologies for assessing feature quality and downstream performance, with particular attention to OCTree's decision tree reasoning approach that provides language-based feedback for iterative improvements. Current challenges include hallucination, computational efficiency, and domain adaptation. As of March 2025, emerging approaches include inference-time compute scaling, reinforcement learning, and supervised fine-tuning with model distillation. Future directions point toward multimodal feature generation, self-improving systems, and neuro-symbolic approaches. This paper provides a detailed overview of an emerging field that promises to automate and enhance feature engineering through language model reasoning.",
      "arxiv_id": "2503.11989v2",
      "published": "2025-03-15",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2503.11989v2",
      "pdf_url": "https://arxiv.org/pdf/2503.11989v2.pdf"
    },
    {
      "title": "Reasoning Models Hallucinate More: Factuality-Aware Reinforcement Learning for Large Reasoning Models",
      "authors": [
        "Junyi Li",
        "Hwee Tou Ng"
      ],
      "summary": "Large language models (LLMs) have significantly advanced in reasoning tasks through reinforcement learning (RL) optimization, achieving impressive capabilities across various challenging benchmarks. However, our empirical analysis reveals a critical drawback: reasoning-oriented RL fine-tuning significantly increases the prevalence of hallucinations. We theoretically analyze the RL training dynamics, identifying high-variance gradient, entropy-induced randomness, and susceptibility to spurious local optima as key factors leading to hallucinations. To address this drawback, we propose Factuality-aware Step-wise Policy Optimization (FSPO), an innovative RL fine-tuning algorithm incorporating explicit factuality verification at each reasoning step. FSPO leverages automated verification against given evidence to dynamically adjust token-level advantage values, incentivizing factual correctness throughout the reasoning process. Experiments across mathematical reasoning and hallucination benchmarks using Qwen2.5 and Llama models demonstrate that FSPO effectively reduces hallucinations while enhancing reasoning accuracy, substantially improving both reliability and performance.",
      "arxiv_id": "2505.24630v2",
      "published": "2025-05-30",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2505.24630v2",
      "pdf_url": "https://arxiv.org/pdf/2505.24630v2.pdf"
    },
    {
      "title": "Revisiting Entropy in Reinforcement Learning for Large Reasoning Models",
      "authors": [
        "Renren Jin",
        "Pengzhi Gao",
        "Yuqi Ren",
        "Zhuowen Han",
        "Tongxuan Zhang",
        "Wuwei Huang",
        "Wei Liu",
        "Jian Luan",
        "Deyi Xiong"
      ],
      "summary": "Reinforcement learning with verifiable rewards (RLVR) has emerged as a predominant approach for enhancing the reasoning capabilities of large language models (LLMs). However, the entropy of LLMs usually collapses during RLVR training, causing premature convergence to suboptimal local minima and hinder further performance improvement. Although various approaches have been proposed to mitigate entropy collapse, a comprehensive study of entropy in RLVR remains lacking. To address this gap, we conduct extensive experiments to investigate the entropy dynamics of LLMs trained with RLVR and analyze how model entropy correlates with response diversity, calibration, and performance across various benchmarks. Our findings reveal that the number of off-policy updates, the diversity of training data, and the clipping thresholds in the optimization objective are critical factors influencing the entropy of LLMs trained with RLVR. Moreover, we theoretically and empirically demonstrate that tokens with positive advantages are the primary contributors to entropy collapse, and that model entropy can be effectively regulated by adjusting the relative loss weights of tokens with positive and negative advantages during training.",
      "arxiv_id": "2511.05993v1",
      "published": "2025-11-08",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2511.05993v1",
      "pdf_url": "https://arxiv.org/pdf/2511.05993v1.pdf"
    },
    {
      "title": "You Need Reasoning to Learn Reasoning: The Limitations of Label-Free RL in Weak Base Models",
      "authors": [
        "Shuvendu Roy",
        "Hossein Hajimirsadeghi",
        "Mengyao Zhai",
        "Golnoosh Samei"
      ],
      "summary": "Recent advances in large language models have demonstrated the promise of unsupervised reinforcement learning (RL) methods for enhancing reasoning capabilities without external supervision. However, the generalizability of these label-free RL approaches to smaller base models with limited reasoning capabilities remains unexplored. In this work, we systematically investigate the performance of label-free RL methods across different model sizes and reasoning strengths, from 0.5B to 7B parameters. Our empirical analysis reveals critical limitations: label-free RL is highly dependent on the base model's pre-existing reasoning capability, with performance often degrading below baseline levels for weaker models. We find that smaller models fail to generate sufficiently long or diverse chain-of-thought reasoning to enable effective self-reflection, and that training data difficulty plays a crucial role in determining success. To address these challenges, we propose a simple yet effective method for label-free RL that utilizes curriculum learning to progressively introduce harder problems during training and mask no-majority rollouts during training. Additionally, we introduce a data curation pipeline to generate samples with predefined difficulty. Our approach demonstrates consistent improvements across all model sizes and reasoning capabilities, providing a path toward more robust unsupervised RL that can bootstrap reasoning abilities in resource-constrained models. We make our code available at https://github.com/BorealisAI/CuMa",
      "arxiv_id": "2511.04902v1",
      "published": "2025-11-07",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2511.04902v1",
      "pdf_url": "https://arxiv.org/pdf/2511.04902v1.pdf"
    },
    {
      "title": "Phi-4-Mini-Reasoning: Exploring the Limits of Small Reasoning Language Models in Math",
      "authors": [
        "Haoran Xu",
        "Baolin Peng",
        "Hany Awadalla",
        "Dongdong Chen",
        "Yen-Chun Chen",
        "Mei Gao",
        "Young Jin Kim",
        "Yunsheng Li",
        "Liliang Ren",
        "Yelong Shen",
        "Shuohang Wang",
        "Weijian Xu",
        "Jianfeng Gao",
        "Weizhu Chen"
      ],
      "summary": "Chain-of-Thought (CoT) significantly enhances formal reasoning capabilities in Large Language Models (LLMs) by training them to explicitly generate intermediate reasoning steps. While LLMs readily benefit from such techniques, improving reasoning in Small Language Models (SLMs) remains challenging due to their limited model capacity. Recent work by Deepseek-R1 demonstrates that distillation from LLM-generated synthetic data can substantially improve the reasoning ability of SLM. However, the detailed modeling recipe is not disclosed. In this work, we present a systematic training recipe for SLMs that consists of four steps: (1) large-scale mid-training on diverse distilled long-CoT data, (2) supervised fine-tuning on high-quality long-CoT data, (3) Rollout DPO leveraging a carefully curated preference dataset, and (4) Reinforcement Learning (RL) with Verifiable Reward. We apply our method on Phi-4-Mini, a compact 3.8B-parameter model. The resulting Phi-4-Mini-Reasoning model exceeds, on math reasoning tasks, much larger reasoning models, e.g., outperforming DeepSeek-R1-Distill-Qwen-7B by 3.2 points and DeepSeek-R1-Distill-Llama-8B by 7.7 points on Math-500. Our results validate that a carefully designed training recipe, with large-scale high-quality CoT data, is effective to unlock strong reasoning capabilities even in resource-constrained small models.",
      "arxiv_id": "2504.21233v1",
      "published": "2025-04-30",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2504.21233v1",
      "pdf_url": "https://arxiv.org/pdf/2504.21233v1.pdf"
    },
    {
      "title": "INTELLECT-2: A Reasoning Model Trained Through Globally Decentralized Reinforcement Learning",
      "authors": [
        " Prime Intellect Team",
        "Sami Jaghouar",
        "Justus Mattern",
        "Jack Min Ong",
        "Jannik Straube",
        "Manveer Basra",
        "Aaron Pazdera",
        "Kushal Thaman",
        "Matthew Di Ferrante",
        "Felix Gabriel",
        "Fares Obeid",
        "Kemal Erdem",
        "Michael Keiblinger",
        "Johannes Hagemann"
      ],
      "summary": "We introduce INTELLECT-2, the first globally distributed reinforcement learning (RL) training run of a 32 billion parameter language model. Unlike traditional centralized training efforts, INTELLECT-2 trains a reasoning model using fully asynchronous RL across a dynamic, heterogeneous swarm of permissionless compute contributors.   To enable a training run with this unique infrastructure, we built various components from scratch: we introduce PRIME-RL, our training framework purpose-built for distributed asynchronous reinforcement learning, based on top of novel components such as TOPLOC, which verifies rollouts from untrusted inference workers, and SHARDCAST, which efficiently broadcasts policy weights from training nodes to inference workers.   Beyond infrastructure components, we propose modifications to the standard GRPO training recipe and data filtering techniques that were crucial to achieve training stability and ensure that our model successfully learned its training objective, thus improving upon QwQ-32B, the state of the art reasoning model in the 32B parameter range.   We open-source INTELLECT-2 along with all of our code and data, hoping to encourage and enable more open research in the field of decentralized training.",
      "arxiv_id": "2505.07291v1",
      "published": "2025-05-12",
      "categories": [
        "cs.LG",
        "cs.DC"
      ],
      "url": "https://arxiv.org/abs/2505.07291v1",
      "pdf_url": "https://arxiv.org/pdf/2505.07291v1.pdf"
    },
    {
      "title": "GEMeX-RMCoT: An Enhanced Med-VQA Dataset for Region-Aware Multimodal Chain-of-Thought Reasoning",
      "authors": [
        "Bo Liu",
        "Xiangyu Zhao",
        "Along He",
        "Yidi Chen",
        "Huazhu Fu",
        "Xiao-Ming Wu"
      ],
      "summary": "Medical visual question answering aims to support clinical decision-making by enabling models to answer natural language questions based on medical images. While recent advances in multi-modal learning have significantly improved performance, current methods still suffer from limited answer reliability and poor interpretability, impairing the ability of clinicians and patients to understand and trust model outputs. To address these limitations, this work first proposes a Region-Aware Multimodal Chain-of-Thought (RMCoT) dataset, in which the process of producing an answer is preceded by a sequence of intermediate reasoning steps that explicitly ground relevant visual regions of the medical image, thereby providing fine-grained explainability. Furthermore, we introduce a novel verifiable reward mechanism for reinforcement learning to guide post-training, improving the alignment between the model's reasoning process and its final answer. Remarkably, our method achieves comparable performance using only one-eighth of the training data, demonstrating the efficiency and effectiveness of the proposal. The dataset is available at https://www.med-vqa.com/GEMeX/.",
      "arxiv_id": "2506.17939v2",
      "published": "2025-06-22",
      "categories": [
        "cs.CV",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.17939v2",
      "pdf_url": "https://arxiv.org/pdf/2506.17939v2.pdf"
    },
    {
      "title": "ReSearch: Learning to Reason with Search for LLMs via Reinforcement Learning",
      "authors": [
        "Mingyang Chen",
        "Linzhuang Sun",
        "Tianpeng Li",
        "Haoze Sun",
        "Yijie Zhou",
        "Chenzheng Zhu",
        "Haofen Wang",
        "Jeff Z. Pan",
        "Wen Zhang",
        "Huajun Chen",
        "Fan Yang",
        "Zenan Zhou",
        "Weipeng Chen"
      ],
      "summary": "Large Language Models (LLMs) have shown remarkable capabilities in reasoning, exemplified by the success of OpenAI-o1 and DeepSeek-R1. However, integrating reasoning with external search processes remains challenging, especially for complex multi-hop questions requiring multiple retrieval steps. We propose ReSearch, a novel framework that trains LLMs to Reason with Search via reinforcement learning without using any supervised data on reasoning steps. Our approach treats search operations as integral components of the reasoning chain, where when and how to perform searches is guided by text-based thinking, and search results subsequently influence further reasoning. We train ReSearch on Qwen2.5-7B(-Instruct) and Qwen2.5-32B(-Instruct) models and conduct extensive experiments. Despite being trained on only one dataset, our models demonstrate strong generalizability across various benchmarks. Analysis reveals that ReSearch naturally elicits advanced reasoning capabilities such as reflection and self-correction during the reinforcement learning process.",
      "arxiv_id": "2503.19470v3",
      "published": "2025-03-25",
      "categories": [
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2503.19470v3",
      "pdf_url": "https://arxiv.org/pdf/2503.19470v3.pdf"
    },
    {
      "title": "Advancing Reasoning in Large Language Models: Promising Methods and Approaches",
      "authors": [
        "Avinash Patil",
        "Aryan Jadon"
      ],
      "summary": "Large Language Models (LLMs) have succeeded remarkably in various natural language processing (NLP) tasks, yet their reasoning capabilities remain a fundamental challenge. While LLMs exhibit impressive fluency and factual recall, their ability to perform complex reasoning-spanning logical deduction, mathematical problem-solving, commonsense inference, and multi-step reasoning-often falls short of human expectations. This survey provides a comprehensive review of emerging techniques enhancing reasoning in LLMs. We categorize existing methods into key approaches, including prompting strategies (e.g., Chain-of-Thought reasoning, Self-Consistency, and Tree-of-Thought reasoning), architectural innovations (e.g., retrieval-augmented models, modular reasoning networks, and neuro-symbolic integration), and learning paradigms (e.g., fine-tuning with reasoning-specific datasets, reinforcement learning, and self-supervised reasoning objectives). Additionally, we explore evaluation frameworks used to assess reasoning in LLMs and highlight open challenges, such as hallucinations, robustness, and reasoning generalization across diverse tasks. By synthesizing recent advancements, this survey aims to provide insights into promising directions for future research and practical applications of reasoning-augmented LLMs.",
      "arxiv_id": "2502.03671v2",
      "published": "2025-02-05",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2502.03671v2",
      "pdf_url": "https://arxiv.org/pdf/2502.03671v2.pdf"
    },
    {
      "title": "DYSTIL: Dynamic Strategy Induction with Large Language Models for Reinforcement Learning",
      "authors": [
        "Borui Wang",
        "Kathleen McKeown",
        "Rex Ying"
      ],
      "summary": "Reinforcement learning from expert demonstrations has long remained a challenging research problem, and existing state-of-the-art methods using behavioral cloning plus further RL training often suffer from poor generalization, low sample efficiency, and poor model interpretability. Inspired by the strong reasoning abilities of large language models (LLMs), we propose a novel strategy-based reinforcement learning framework integrated with LLMs called DYnamic STrategy Induction with Llms for reinforcement learning (DYSTIL) to overcome these limitations. DYSTIL dynamically queries a strategy-generating LLM to induce textual strategies based on advantage estimations and expert demonstrations, and gradually internalizes induced strategies into the RL agent through policy optimization to improve its performance through boosting policy generalization and enhancing sample efficiency. It also provides a direct textual channel to observe and interpret the evolution of the policy's underlying strategies during training. We test DYSTIL over challenging RL environments from Minigrid and BabyAI, and empirically demonstrate that DYSTIL significantly outperforms state-of-the-art baseline methods by 17.75% in average success rate while also enjoying higher sample efficiency during the learning process.",
      "arxiv_id": "2505.03209v1",
      "published": "2025-05-06",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2505.03209v1",
      "pdf_url": "https://arxiv.org/pdf/2505.03209v1.pdf"
    },
    {
      "title": "Think Before You Act: Unified Policy for Interleaving Language Reasoning with Actions",
      "authors": [
        "Lina Mezghani",
        "Piotr Bojanowski",
        "Karteek Alahari",
        "Sainbayar Sukhbaatar"
      ],
      "summary": "The success of transformer models trained with a language modeling objective brings a promising opportunity to the reinforcement learning framework. Decision Transformer is a step towards this direction, showing how to train transformers with a similar next-step prediction objective on offline data. Another important development in this area is the recent emergence of large-scale datasets collected from the internet, such as the ones composed of tutorial videos with captions where people talk about what they are doing. To take advantage of this language component, we propose a novel method for unifying language reasoning with actions in a single policy. Specifically, we augment a transformer policy with word outputs, so it can generate textual captions interleaved with actions. When tested on the most challenging task in BabyAI, with captions describing next subgoals, our reasoning policy consistently outperforms the caption-free baseline.",
      "arxiv_id": "2304.11063v1",
      "published": "2023-04-18",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2304.11063v1",
      "pdf_url": "https://arxiv.org/pdf/2304.11063v1.pdf"
    },
    {
      "title": "Mitigating Forgetting Between Supervised and Reinforcement Learning Yields Stronger Reasoners",
      "authors": [
        "Xiangchi Yuan",
        "Xiang Chen",
        "Tong Yu",
        "Dachuan Shi",
        "Can Jin",
        "Wenke Lee",
        "Saayan Mitra"
      ],
      "summary": "Large Language Models (LLMs) show strong reasoning abilities, often amplified by Chain-of-Thought (CoT) prompting and reinforcement learning (RL). Although RL algorithms can substantially improve reasoning, they struggle to expand reasoning boundaries because they learn from their own reasoning trajectories rather than acquiring external knowledge. Supervised fine-tuning (SFT) offers complementary benefits but typically requires large-scale data and risks overfitting. Recent attempts to combine SFT and RL face three main challenges: data inefficiency, algorithm-specific designs, and catastrophic forgetting. We propose a plug-and-play framework that dynamically integrates SFT into RL by selecting challenging examples for SFT. This approach reduces SFT data requirements and remains agnostic to the choice of RL or SFT algorithm. To mitigate catastrophic forgetting of RL-acquired skills during SFT, we select high-entropy tokens for loss calculation and freeze parameters identified as critical for RL. Our method achieves state-of-the-art (SoTA) reasoning performance using only 1.5% of the SFT data and 20.4% of the RL data used by prior SoTA, providing an efficient and plug-and-play solution for combining SFT and RL in reasoning post-training.",
      "arxiv_id": "2510.04454v1",
      "published": "2025-10-06",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.04454v1",
      "pdf_url": "https://arxiv.org/pdf/2510.04454v1.pdf"
    },
    {
      "title": "Enhancing Reasoning Skills in Small Persian Medical Language Models Can Outperform Large-Scale Data Training",
      "authors": [
        "Mehrdad Ghassabi",
        "Sadra Hakim",
        "Hamidreza Baradaran Kashani",
        "Pedram Rostami"
      ],
      "summary": "Enhancing reasoning capabilities in small language models is critical for specialized applications such as medical question answering, particularly in underrepresented languages like Persian. In this study, we employ Reinforcement Learning with AI Feedback (RLAIF) and Direct preference optimization (DPO) to improve the reasoning skills of a general-purpose Persian language model. To achieve this, we translated a multiple-choice medical question-answering dataset into Persian and used RLAIF to generate rejected-preferred answer pairs, which are essential for DPO training. By prompting both teacher and student models to produce Chain-of-Thought (CoT) reasoning responses, we compiled a dataset containing correct and incorrect reasoning trajectories. This dataset, comprising 2 million tokens in preferred answers and 2.5 million tokens in rejected ones, was used to train a baseline model, significantly enhancing its medical reasoning capabilities in Persian. Remarkably, the resulting model outperformed its predecessor, gaokerena-V, which was trained on approximately 57 million tokens, despite leveraging a much smaller dataset. These results highlight the efficiency and effectiveness of reasoning-focused training approaches in developing domain-specific language models with limited data availability.",
      "arxiv_id": "2510.20059v2",
      "published": "2025-10-22",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.20059v2",
      "pdf_url": "https://arxiv.org/pdf/2510.20059v2.pdf"
    },
    {
      "title": "Hybrid Latent Reasoning via Reinforcement Learning",
      "authors": [
        "Zhenrui Yue",
        "Bowen Jin",
        "Huimin Zeng",
        "Honglei Zhuang",
        "Zhen Qin",
        "Jinsung Yoon",
        "Lanyu Shang",
        "Jiawei Han",
        "Dong Wang"
      ],
      "summary": "Recent advances in large language models (LLMs) have introduced latent reasoning as a promising alternative to autoregressive reasoning. By performing internal computation with hidden states from previous steps, latent reasoning benefit from more informative features rather than sampling a discrete chain-of-thought (CoT) path. Yet latent reasoning approaches are often incompatible with LLMs, as their continuous paradigm conflicts with the discrete nature of autoregressive generation. Moreover, these methods rely on CoT traces for training and thus fail to exploit the inherent reasoning patterns of LLMs. In this work, we explore latent reasoning by leveraging the intrinsic capabilities of LLMs via reinforcement learning (RL). To this end, we introduce hybrid reasoning policy optimization (HRPO), an RL-based hybrid latent reasoning approach that (1) integrates prior hidden states into sampled tokens with a learnable gating mechanism, and (2) initializes training with predominantly token embeddings while progressively incorporating more hidden features. This design maintains LLMs' generative capabilities and incentivizes hybrid reasoning using both discrete and continuous representations. In addition, the hybrid HRPO introduces stochasticity into latent reasoning via token sampling, thereby enabling RL-based optimization without requiring CoT trajectories. Extensive evaluations across diverse benchmarks show that HRPO outperforms prior methods in both knowledge- and reasoning-intensive tasks. Furthermore, HRPO-trained LLMs remain interpretable and exhibit intriguing behaviors like cross-lingual patterns and shorter completion lengths, highlighting the potential of our RL-based approach and offer insights for future work in latent reasoning.",
      "arxiv_id": "2505.18454v2",
      "published": "2025-05-24",
      "categories": [
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2505.18454v2",
      "pdf_url": "https://arxiv.org/pdf/2505.18454v2.pdf"
    },
    {
      "title": "Think or Not? Selective Reasoning via Reinforcement Learning for Vision-Language Models",
      "authors": [
        "Jiaqi Wang",
        "Kevin Qinghong Lin",
        "James Cheng",
        "Mike Zheng Shou"
      ],
      "summary": "Reinforcement Learning (RL) has proven to be an effective post-training strategy for enhancing reasoning in vision-language models (VLMs). Group Relative Policy Optimization (GRPO) is a recent prominent method that encourages models to generate complete reasoning traces before answering, leading to increased token usage and computational cost. Inspired by the human-like thinking process-where people skip reasoning for easy questions but think carefully when needed-we explore how to enable VLMs to first decide when reasoning is necessary. To realize this, we propose TON, a two-stage training strategy: (i) a supervised fine-tuning (SFT) stage with a simple yet effective 'thought dropout' operation, where reasoning traces are randomly replaced with empty thoughts. This introduces a think-or-not format that serves as a cold start for selective reasoning; (ii) a GRPO stage that enables the model to freely explore when to think or not, while maximizing task-aware outcome rewards. Experimental results show that TON can reduce the completion length by up to 90% compared to vanilla GRPO, without sacrificing performance or even improving it. Further evaluations across LLM (GSM8K), VLM (CLEVR, Super-CLEVR, GeoQA), and Agentic (AITZ) tasks-covering a range of reasoning difficulties under both 3B and 7B models-consistently reveal that the model progressively learns to bypass unnecessary reasoning steps as training advances. These findings shed light on the path toward human-like reasoning patterns in RL approaches. Our code is available at https://github.com/kokolerk/TON.",
      "arxiv_id": "2505.16854v3",
      "published": "2025-05-22",
      "categories": [
        "cs.AI",
        "cs.CV"
      ],
      "url": "https://arxiv.org/abs/2505.16854v3",
      "pdf_url": "https://arxiv.org/pdf/2505.16854v3.pdf"
    },
    {
      "title": "Dual-Weighted Reinforcement Learning for Generative Preference Modeling",
      "authors": [
        "Shengyu Feng",
        "Yun He",
        "Shuang Ma",
        "Beibin Li",
        "Yuanhao Xiong",
        "Songlin Li",
        "Karishma Mandyam",
        "Julian Katz-Samuels",
        "Shengjie Bi",
        "Licheng Yu",
        "Hejia Zhang",
        "Karthik Abinav Sankararaman",
        "Han Fang",
        "Riham Mansour",
        "Yiming Yang",
        "Manaal Faruqui"
      ],
      "summary": "Reinforcement learning (RL) has recently proven effective at scaling chain-of-thought (CoT) reasoning in large language models on tasks with verifiable answers. However, extending RL to more general non-verifiable tasks, typically in the format of human preference pairs, remains both challenging and underexplored. In this work, we propose Dual-Weighted Reinforcement Learning (DWRL), a new framework for preference modeling that integrates CoT reasoning with the Bradley-Terry (BT) model via a dual-weighted RL objective that preserves preference-modeling inductive bias. DWRL approximates the maximum-likelihood objective of the BT model with two complementary weights: an instance-wise misalignment weight, which emphasizes under-trained pairs misaligned with human preference, and a group-wise (self-normalized) conditional preference score, which promotes promising thoughts. In this paper, we apply DWRL to preference modeling by training generative preference models (GPMs) to first generate a thought and then predict the human preference score. Across multiple benchmarks and model scales (Llama3 and Qwen2.5), DWRL consistently outperforms both GPM baselines and scalar models, while producing coherent, interpretable thoughts. In summary, our results position DWRL as a general framework for reasoning-enhanced preference learning beyond verifiable tasks.",
      "arxiv_id": "2510.15242v2",
      "published": "2025-10-17",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2510.15242v2",
      "pdf_url": "https://arxiv.org/pdf/2510.15242v2.pdf"
    },
    {
      "title": "Teaching Large Language Models to Reason with Reinforcement Learning",
      "authors": [
        "Alex Havrilla",
        "Yuqing Du",
        "Sharath Chandra Raparthy",
        "Christoforos Nalmpantis",
        "Jane Dwivedi-Yu",
        "Maksym Zhuravinskyi",
        "Eric Hambro",
        "Sainbayar Sukhbaatar",
        "Roberta Raileanu"
      ],
      "summary": "Reinforcement Learning from Human Feedback (\\textbf{RLHF}) has emerged as a dominant approach for aligning LLM outputs with human preferences. Inspired by the success of RLHF, we study the performance of multiple algorithms that learn from feedback (Expert Iteration, Proximal Policy Optimization (\\textbf{PPO}), Return-Conditioned RL) on improving LLM reasoning capabilities. We investigate both sparse and dense rewards provided to the LLM both heuristically and via a learned reward model. We additionally start from multiple model sizes and initializations both with and without supervised fine-tuning (\\textbf{SFT}) data. Overall, we find all algorithms perform comparably, with Expert Iteration performing best in most cases. Surprisingly, we find the sample complexity of Expert Iteration is similar to that of PPO, requiring at most on the order of $10^6$ samples to converge from a pretrained checkpoint. We investigate why this is the case, concluding that during RL training models fail to explore significantly beyond solutions already produced by SFT models. Additionally, we discuss a trade off between maj@1 and pass@96 metric performance during SFT training and how conversely RL training improves both simultaneously. We then conclude by discussing the implications of our findings for RLHF and the future role of RL in LLM fine-tuning.",
      "arxiv_id": "2403.04642v1",
      "published": "2024-03-07",
      "categories": [
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2403.04642v1",
      "pdf_url": "https://arxiv.org/pdf/2403.04642v1.pdf"
    }
  ],
  "alphaco_related": [
    {
      "title": "An Empirical Evaluation of Competitive Programming AI: A Case Study of AlphaCode",
      "authors": [
        "Sila Lertbanjongngam",
        "Bodin Chinthanet",
        "Takashi Ishio",
        "Raula Gaikovina Kula",
        "Pattara Leelaprute",
        "Bundit Manaskasemsak",
        "Arnon Rungsawang",
        "Kenichi Matsumoto"
      ],
      "summary": "AlphaCode is a code generation system for assisting software developers in solving competitive programming problems using natural language problem descriptions. Despite the advantages of the code generating system, the open source community expressed concerns about practicality and data licensing. However, there is no research investigating generated codes in terms of code clone and performance. In this paper, we conduct an empirical study to find code similarities and performance differences between AlphaCode-generated codes and human codes. The results show that (i) the generated codes from AlphaCode are similar to human codes (i.e., the average maximum similarity score is 0.56) and (ii) the generated code performs on par with or worse than the human code in terms of execution time and memory usage. Moreover, AlphaCode tends to generate more similar codes to humans for low-difficulty problems (i.e., four cases have the exact same codes). It also employs excessive nested loops and unnecessary variable declarations for high-difficulty problems, which cause low performance regarding our manual investigation. The replication package is available at https:/doi.org/10.5281/zenodo.6820681",
      "arxiv_id": "2208.08603v2",
      "published": "2022-08-18",
      "categories": [
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2208.08603v2",
      "pdf_url": "https://arxiv.org/pdf/2208.08603v2.pdf"
    },
    {
      "title": "Competition-Level Code Generation with AlphaCode",
      "authors": [
        "Yujia Li",
        "David Choi",
        "Junyoung Chung",
        "Nate Kushman",
        "Julian Schrittwieser",
        "R\u00e9mi Leblond",
        "Tom Eccles",
        "James Keeling",
        "Felix Gimeno",
        "Agustin Dal Lago",
        "Thomas Hubert",
        "Peter Choy",
        "Cyprien de Masson d'Autume",
        "Igor Babuschkin",
        "Xinyun Chen",
        "Po-Sen Huang",
        "Johannes Welbl",
        "Sven Gowal",
        "Alexey Cherepanov",
        "James Molloy",
        "Daniel J. Mankowitz",
        "Esme Sutherland Robson",
        "Pushmeet Kohli",
        "Nando de Freitas",
        "Koray Kavukcuoglu",
        "Oriol Vinyals"
      ],
      "summary": "Programming is a powerful and ubiquitous problem-solving tool. Developing systems that can assist programmers or even generate programs independently could make programming more productive and accessible, yet so far incorporating innovations in AI has proven challenging. Recent large-scale language models have demonstrated an impressive ability to generate code, and are now able to complete simple programming tasks. However, these models still perform poorly when evaluated on more complex, unseen problems that require problem-solving skills beyond simply translating instructions into code. For example, competitive programming problems which require an understanding of algorithms and complex natural language remain extremely challenging. To address this gap, we introduce AlphaCode, a system for code generation that can create novel solutions to these problems that require deeper reasoning. In simulated evaluations on recent programming competitions on the Codeforces platform, AlphaCode achieved on average a ranking of top 54.3% in competitions with more than 5,000 participants. We found that three key components were critical to achieve good and reliable performance: (1) an extensive and clean competitive programming dataset for training and evaluation, (2) large and efficient-to-sample transformer-based architectures, and (3) large-scale model sampling to explore the search space, followed by filtering based on program behavior to a small set of submissions.",
      "arxiv_id": "2203.07814v1",
      "published": "2022-02-08",
      "categories": [
        "cs.PL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2203.07814v1",
      "pdf_url": "https://arxiv.org/pdf/2203.07814v1.pdf"
    },
    {
      "title": "CodeRL+: Improving Code Generation via Reinforcement with Execution Semantics Alignment",
      "authors": [
        "Xue Jiang",
        "Yihong Dong",
        "Mengyang Liu",
        "Hongyi Deng",
        "Tian Wang",
        "Yongding Tao",
        "Rongyu Cao",
        "Binhua Li",
        "Zhi Jin",
        "Wenpin Jiao",
        "Fei Huang",
        "Yongbin Li",
        "Ge Li"
      ],
      "summary": "While Large Language Models (LLMs) excel at code generation by learning from vast code corpora, a fundamental semantic gap remains between their training on textual patterns and the goal of functional correctness, which is governed by formal execution semantics. Reinforcement Learning with Verifiable Rewards (RLVR) approaches attempt to bridge this gap using outcome rewards from executing test cases. However, solely relying on binary pass/fail signals is inefficient for establishing a well-aligned connection between the textual representation of code and its execution semantics, especially for subtle logical errors within the code. In this paper, we propose CodeRL+, a novel approach that integrates execution semantics alignment into the RLVR training pipeline for code generation. CodeRL+ enables the model to infer variable-level execution trajectory, providing a direct learning signal of execution semantics. CodeRL+ can construct execution semantics alignment directly using existing on-policy rollouts and integrates seamlessly with various RL algorithms. Extensive experiments demonstrate that CodeRL+ outperforms post-training baselines (including RLVR and Distillation), achieving a 4.6% average relative improvement in pass@1. CodeRL+ generalizes effectively to other coding tasks, yielding 15.5% and 4.4% higher accuracy on code-reasoning and test-output-generation benchmarks, respectively. CodeRL+ shows strong applicability across diverse RL algorithms and LLMs. Furthermore, probe analyses provide compelling evidence that CodeRL+ strengthens the alignment between code's textual representations and its underlying execution semantics.",
      "arxiv_id": "2510.18471v1",
      "published": "2025-10-21",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2510.18471v1",
      "pdf_url": "https://arxiv.org/pdf/2510.18471v1.pdf"
    },
    {
      "title": "Competitive Programming with Large Reasoning Models",
      "authors": [
        " OpenAI",
        " :",
        "Ahmed El-Kishky",
        "Alexander Wei",
        "Andre Saraiva",
        "Borys Minaiev",
        "Daniel Selsam",
        "David Dohan",
        "Francis Song",
        "Hunter Lightman",
        "Ignasi Clavera",
        "Jakub Pachocki",
        "Jerry Tworek",
        "Lorenz Kuhn",
        "Lukasz Kaiser",
        "Mark Chen",
        "Max Schwarzer",
        "Mostafa Rohaninejad",
        "Nat McAleese",
        "o3 contributors",
        "Oleg M\u00fcrk",
        "Rhythm Garg",
        "Rui Shu",
        "Szymon Sidor",
        "Vineet Kosaraju",
        "Wenda Zhou"
      ],
      "summary": "We show that reinforcement learning applied to large language models (LLMs) significantly boosts performance on complex coding and reasoning tasks. Additionally, we compare two general-purpose reasoning models - OpenAI o1 and an early checkpoint of o3 - with a domain-specific system, o1-ioi, which uses hand-engineered inference strategies designed for competing in the 2024 International Olympiad in Informatics (IOI). We competed live at IOI 2024 with o1-ioi and, using hand-crafted test-time strategies, placed in the 49th percentile. Under relaxed competition constraints, o1-ioi achieved a gold medal. However, when evaluating later models such as o3, we find that o3 achieves gold without hand-crafted domain-specific strategies or relaxed constraints. Our findings show that although specialized pipelines such as o1-ioi yield solid improvements, the scaled-up, general-purpose o3 model surpasses those results without relying on hand-crafted inference heuristics. Notably, o3 achieves a gold medal at the 2024 IOI and obtains a Codeforces rating on par with elite human competitors. Overall, these results indicate that scaling general-purpose reinforcement learning, rather than relying on domain-specific techniques, offers a robust path toward state-of-the-art AI in reasoning domains, such as competitive programming.",
      "arxiv_id": "2502.06807v2",
      "published": "2025-02-03",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2502.06807v2",
      "pdf_url": "https://arxiv.org/pdf/2502.06807v2.pdf"
    },
    {
      "title": "An Improved Reinforcement Learning Algorithm for Learning to Branch",
      "authors": [
        "Qingyu Qu",
        "Xijun Li",
        "Yunfan Zhou",
        "Jia Zeng",
        "Mingxuan Yuan",
        "Jie Wang",
        "Jinhu Lv",
        "Kexin Liu",
        "Kun Mao"
      ],
      "summary": "Most combinatorial optimization problems can be formulated as mixed integer linear programming (MILP), in which branch-and-bound (B\\&B) is a general and widely used method. Recently, learning to branch has become a hot research topic in the intersection of machine learning and combinatorial optimization. In this paper, we propose a novel reinforcement learning-based B\\&B algorithm. Similar to offline reinforcement learning, we initially train on the demonstration data to accelerate learning massively. With the improvement of the training effect, the agent starts to interact with the environment with its learned policy gradually. It is critical to improve the performance of the algorithm by determining the mixing ratio between demonstration and self-generated data. Thus, we propose a prioritized storage mechanism to control this ratio automatically. In order to improve the robustness of the training process, a superior network is additionally introduced based on Double DQN, which always serves as a Q-network with competitive performance. We evaluate the performance of the proposed algorithm over three public research benchmarks and compare it against strong baselines, including three classical heuristics and one state-of-the-art imitation learning-based branching algorithm. The results show that the proposed algorithm achieves the best performance among compared algorithms and possesses the potential to improve B\\&B algorithm performance continuously.",
      "arxiv_id": "2201.06213v1",
      "published": "2022-01-17",
      "categories": [
        "cs.LG",
        "cs.AI",
        "math.OC"
      ],
      "url": "https://arxiv.org/abs/2201.06213v1",
      "pdf_url": "https://arxiv.org/pdf/2201.06213v1.pdf"
    },
    {
      "title": "CodeContests+: High-Quality Test Case Generation for Competitive Programming",
      "authors": [
        "Zihan Wang",
        "Siyao Liu",
        "Yang Sun",
        "Hongyan Li",
        "Kai Shen"
      ],
      "summary": "Competitive programming, due to its high reasoning difficulty and precise correctness feedback, has become a key task for both training and evaluating the reasoning capabilities of large language models (LLMs). However, while a large amount of public problem data, such as problem statements and solutions, is available, the test cases of these problems are often difficult to obtain. Therefore, test case generation is a necessary task for building large-scale datasets, and the quality of the test cases directly determines the accuracy of the evaluation. In this paper, we introduce an LLM-based agent system that creates high-quality test cases for competitive programming problems. We apply this system to the CodeContests dataset and propose a new version with improved test cases, named CodeContests+. We evaluated the quality of test cases in CodeContestsPlus. First, we used 1.72 million submissions with pass/fail labels to examine the accuracy of these test cases in evaluation. The results indicated that CodeContests+ achieves significantly higher accuracy than CodeContests, particularly with a notably higher True Positive Rate (TPR). Subsequently, our experiments in LLM Reinforcement Learning (RL) further confirmed that improvements in test case quality yield considerable advantages for RL.",
      "arxiv_id": "2506.05817v1",
      "published": "2025-06-06",
      "categories": [
        "cs.SE",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2506.05817v1",
      "pdf_url": "https://arxiv.org/pdf/2506.05817v1.pdf"
    },
    {
      "title": "Programming with AI: Evaluating ChatGPT, Gemini, AlphaCode, and GitHub Copilot for Programmers",
      "authors": [
        "Md Kamrul Siam",
        "Huanying Gu",
        "Jerry Q. Cheng"
      ],
      "summary": "Our everyday lives now heavily rely on artificial intelligence (AI) powered large language models (LLMs). Like regular users, programmers are also benefiting from the newest large language models. In response to the critical role that AI models play in modern software development, this study presents a thorough evaluation of leading programming assistants, including ChatGPT, Gemini(Bard AI), AlphaCode, and GitHub Copilot. The evaluation is based on tasks like natural language processing and code generation accuracy in different programming languages like Java, Python and C++. Based on the results, it has emphasized their strengths and weaknesses and the importance of further modifications to increase the reliability and accuracy of the latest popular models. Although these AI assistants illustrate a high level of progress in language understanding and code generation, along with ethical considerations and responsible usage, they provoke a necessity for discussion. With time, developing more refined AI technology is essential for achieving advanced solutions in various fields, especially with the knowledge of the feature intricacies of these models and their implications. This study offers a comparison of different LLMs and provides essential feedback on the rapidly changing area of AI models. It also emphasizes the need for ethical developmental practices to actualize AI models' full potential.",
      "arxiv_id": "2411.09224v1",
      "published": "2024-11-14",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2411.09224v1",
      "pdf_url": "https://arxiv.org/pdf/2411.09224v1.pdf"
    },
    {
      "title": "Graph-based Reinforcement Learning meets Mixed Integer Programs: An application to 3D robot assembly discovery",
      "authors": [
        "Niklas Funk",
        "Svenja Menzenbach",
        "Georgia Chalvatzaki",
        "Jan Peters"
      ],
      "summary": "Robot assembly discovery is a challenging problem that lives at the intersection of resource allocation and motion planning. The goal is to combine a predefined set of objects to form something new while considering task execution with the robot-in-the-loop. In this work, we tackle the problem of building arbitrary, predefined target structures entirely from scratch using a set of Tetris-like building blocks and a robotic manipulator. Our novel hierarchical approach aims at efficiently decomposing the overall task into three feasible levels that benefit mutually from each other. On the high level, we run a classical mixed-integer program for global optimization of block-type selection and the blocks' final poses to recreate the desired shape. Its output is then exploited to efficiently guide the exploration of an underlying reinforcement learning (RL) policy. This RL policy draws its generalization properties from a flexible graph-based representation that is learned through Q-learning and can be refined with search. Moreover, it accounts for the necessary conditions of structural stability and robotic feasibility that cannot be effectively reflected in the previous layer. Lastly, a grasp and motion planner transforms the desired assembly commands into robot joint movements. We demonstrate our proposed method's performance on a set of competitive simulated RAD environments, showcase real-world transfer, and report performance and robustness gains compared to an unstructured end-to-end approach. Videos are available at https://sites.google.com/view/rl-meets-milp .",
      "arxiv_id": "2203.04120v2",
      "published": "2022-03-08",
      "categories": [
        "cs.RO",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2203.04120v2",
      "pdf_url": "https://arxiv.org/pdf/2203.04120v2.pdf"
    },
    {
      "title": "Symbolic Visual Reinforcement Learning: A Scalable Framework with Object-Level Abstraction and Differentiable Expression Search",
      "authors": [
        "Wenqing Zheng",
        "S P Sharan",
        "Zhiwen Fan",
        "Kevin Wang",
        "Yihan Xi",
        "Zhangyang Wang"
      ],
      "summary": "Learning efficient and interpretable policies has been a challenging task in reinforcement learning (RL), particularly in the visual RL setting with complex scenes. While neural networks have achieved competitive performance, the resulting policies are often over-parameterized black boxes that are difficult to interpret and deploy efficiently. More recent symbolic RL frameworks have shown that high-level domain-specific programming logic can be designed to handle both policy learning and symbolic planning. However, these approaches rely on coded primitives with little feature learning, and when applied to high-dimensional visual scenes, they can suffer from scalability issues and perform poorly when images have complex object interactions. To address these challenges, we propose \\textit{Differentiable Symbolic Expression Search} (DiffSES), a novel symbolic learning approach that discovers discrete symbolic policies using partially differentiable optimization. By using object-level abstractions instead of raw pixel-level inputs, DiffSES is able to leverage the simplicity and scalability advantages of symbolic expressions, while also incorporating the strengths of neural networks for feature learning and optimization. Our experiments demonstrate that DiffSES is able to generate symbolic policies that are simpler and more and scalable than state-of-the-art symbolic RL methods, with a reduced amount of symbolic prior knowledge.",
      "arxiv_id": "2212.14849v1",
      "published": "2022-12-30",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2212.14849v1",
      "pdf_url": "https://arxiv.org/pdf/2212.14849v1.pdf"
    },
    {
      "title": "Learning to Stop Cut Generation for Efficient Mixed-Integer Linear Programming",
      "authors": [
        "Haotian Ling",
        "Zhihai Wang",
        "Jie Wang"
      ],
      "summary": "Cutting planes (cuts) play an important role in solving mixed-integer linear programs (MILPs), as they significantly tighten the dual bounds and improve the solving performance. A key problem for cuts is when to stop cuts generation, which is important for the efficiency of solving MILPs. However, many modern MILP solvers employ hard-coded heuristics to tackle this problem, which tends to neglect underlying patterns among MILPs from certain applications. To address this challenge, we formulate the cuts generation stopping problem as a reinforcement learning problem and propose a novel hybrid graph representation model (HYGRO) to learn effective stopping strategies. An appealing feature of HYGRO is that it can effectively capture both the dynamic and static features of MILPs, enabling dynamic decision-making for the stopping strategies. To the best of our knowledge, HYGRO is the first data-driven method to tackle the cuts generation stopping problem. By integrating our approach with modern solvers, experiments demonstrate that HYGRO significantly improves the efficiency of solving MILPs compared to competitive baselines, achieving up to 31% improvement.",
      "arxiv_id": "2401.17527v2",
      "published": "2024-01-31",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2401.17527v2",
      "pdf_url": "https://arxiv.org/pdf/2401.17527v2.pdf"
    },
    {
      "title": "DanZero+: Dominating the GuanDan Game through Reinforcement Learning",
      "authors": [
        "Youpeng Zhao",
        "Yudong Lu",
        "Jian Zhao",
        "Wengang Zhou",
        "Houqiang Li"
      ],
      "summary": "The utilization of artificial intelligence (AI) in card games has been a well-explored subject within AI research for an extensive period. Recent advancements have propelled AI programs to showcase expertise in intricate card games such as Mahjong, DouDizhu, and Texas Hold'em. In this work, we aim to develop an AI program for an exceptionally complex and popular card game called GuanDan. This game involves four players engaging in both competitive and cooperative play throughout a long process to upgrade their level, posing great challenges for AI due to its expansive state and action space, long episode length, and complex rules. Employing reinforcement learning techniques, specifically Deep Monte Carlo (DMC), and a distributed training framework, we first put forward an AI program named DanZero for this game. Evaluation against baseline AI programs based on heuristic rules highlights the outstanding performance of our bot. Besides, in order to further enhance the AI's capabilities, we apply policy-based reinforcement learning algorithm to GuanDan. To address the challenges arising from the huge action space, which will significantly impact the performance of policy-based algorithms, we adopt the pre-trained model to facilitate the training process and the achieved AI program manages to achieve a superior performance.",
      "arxiv_id": "2312.02561v1",
      "published": "2023-12-05",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2312.02561v1",
      "pdf_url": "https://arxiv.org/pdf/2312.02561v1.pdf"
    },
    {
      "title": "Multi-echelon Supply Chains with Uncertain Seasonal Demands and Lead Times Using Deep Reinforcement Learning",
      "authors": [
        "Julio C\u00e9sar Alves",
        "Geraldo Robson Mateus"
      ],
      "summary": "We address the problem of production planning and distribution in multi-echelon supply chains. We consider uncertain demands and lead times which makes the problem stochastic and non-linear. A Markov Decision Process formulation and a Non-linear Programming model are presented. As a sequential decision-making problem, Deep Reinforcement Learning (RL) is a possible solution approach. This type of technique has gained a lot of attention from Artificial Intelligence and Optimization communities in recent years. Considering the good results obtained with Deep RL approaches in different areas there is a growing interest in applying them in problems from the Operations Research field. We have used a Deep RL technique, namely Proximal Policy Optimization (PPO2), to solve the problem considering uncertain, regular and seasonal demands and constant or stochastic lead times. Experiments are carried out in different scenarios to better assess the suitability of the algorithm. An agent based on a linearized model is used as a baseline. Experimental results indicate that PPO2 is a competitive and adequate tool for this type of problem. PPO2 agent is better than baseline in all scenarios with stochastic lead times (7.3-11.2%), regardless of whether demands are seasonal or not. In scenarios with constant lead times, the PPO2 agent is better when uncertain demands are non-seasonal (2.2-4.7%). The results show that the greater the uncertainty of the scenario, the greater the viability of this type of approach.",
      "arxiv_id": "2201.04651v1",
      "published": "2022-01-12",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2201.04651v1",
      "pdf_url": "https://arxiv.org/pdf/2201.04651v1.pdf"
    },
    {
      "title": "A Graph-Enhanced Deep-Reinforcement Learning Framework for the Aircraft Landing Problem",
      "authors": [
        "Vatsal Maru"
      ],
      "summary": "The Aircraft Landing Problem (ALP) is one of the challenging problems in aircraft transportation and management. The challenge is to schedule the arriving aircraft in a sequence so that the cost and delays are optimized. There are various solution approaches to solving this problem, most of which are based on operations research algorithms and meta-heuristics. Although traditional methods perform better on one or the other factors, there remains a problem of solving real-time rescheduling and computational scalability altogether. This paper presents a novel deep reinforcement learning (DRL) framework that combines graph neural networks with actor-critic architectures to address the ALP. This paper introduces three key contributions: A graph-based state representation that efficiently captures temporal and spatial relationships between aircraft, a specialized actor-critic architecture designed to handle multiple competing objectives in landing scheduling, and a runway balance strategy that ensures efficient resource utilization while maintaining safety constraints. The results show that the trained algorithm can be tested on different problem sets and the results are competitive to operation research algorithms. The experimental results on standard benchmark data sets demonstrate a 99.95% reduction in computational time compared to Mixed Integer Programming (MIP) and 38% higher runway throughput over First Come First Serve (FCFS) approaches. Therefore, the proposed solution is competitive to traditional approaches and achieves substantial advancements. Notably, it does not require retraining, making it particularly suitable for industrial deployment. The frameworks capability to generate solutions within 1 second enables real-time rescheduling, addressing critical requirements of air traffic management.",
      "arxiv_id": "2502.12617v2",
      "published": "2025-02-18",
      "categories": [
        "cs.LG",
        "cs.AI",
        "eess.SY"
      ],
      "url": "https://arxiv.org/abs/2502.12617v2",
      "pdf_url": "https://arxiv.org/pdf/2502.12617v2.pdf"
    },
    {
      "title": "Harfang3D Dog-Fight Sandbox: A Reinforcement Learning Research Platform for the Customized Control Tasks of Fighter Aircrafts",
      "authors": [
        "Muhammed Murat \u00d6zbek",
        "S\u00fcleyman Y\u0131ld\u0131r\u0131m",
        "Muhammet Aksoy",
        "Eric Kernin",
        "Emre Koyuncu"
      ],
      "summary": "The advent of deep learning (DL) gave rise to significant breakthroughs in Reinforcement Learning (RL) research. Deep Reinforcement Learning (DRL) algorithms have reached super-human level skills when applied to vision-based control problems as such in Atari 2600 games where environment states were extracted from pixel information. Unfortunately, these environments are far from being applicable to highly dynamic and complex real-world tasks as in autonomous control of a fighter aircraft since these environments only involve 2D representation of a visual world. Here, we present a semi-realistic flight simulation environment Harfang3D Dog-Fight Sandbox for fighter aircrafts. It is aimed to be a flexible toolbox for the investigation of main challenges in aviation studies using Reinforcement Learning. The program provides easy access to flight dynamics model, environment states, and aerodynamics of the plane enabling user to customize any specific task in order to build intelligent decision making (control) systems via RL. The software also allows deployment of bot aircrafts and development of multi-agent tasks. This way, multiple groups of aircrafts can be configured to be competitive or cooperative agents to perform complicated tasks including Dog Fight. During the experiments, we carried out training for two different scenarios: navigating to a designated location and within visual range (WVR) combat, shortly Dog Fight. Using Deep Reinforcement Learning techniques for both scenarios, we were able to train competent agents that exhibit human-like behaviours. Based on this results, it is confirmed that Harfang3D Dog-Fight Sandbox can be utilized as a 3D realistic RL research platform.",
      "arxiv_id": "2210.07282v1",
      "published": "2022-10-13",
      "categories": [
        "cs.RO",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2210.07282v1",
      "pdf_url": "https://arxiv.org/pdf/2210.07282v1.pdf"
    },
    {
      "title": "Meta-learning from Learning Curves Challenge: Lessons learned from the First Round and Design of the Second Round",
      "authors": [
        "Manh Hung Nguyen",
        "Lisheng Sun",
        "Nathan Grinsztajn",
        "Isabelle Guyon"
      ],
      "summary": "Meta-learning from learning curves is an important yet often neglected research area in the Machine Learning community. We introduce a series of Reinforcement Learning-based meta-learning challenges, in which an agent searches for the best suited algorithm for a given dataset, based on feedback of learning curves from the environment. The first round attracted participants both from academia and industry. This paper analyzes the results of the first round (accepted to the competition program of WCCI 2022), to draw insights into what makes a meta-learner successful at learning from learning curves. With the lessons learned from the first round and the feedback from the participants, we have designed the second round of our challenge with a new protocol and a new meta-dataset. The second round of our challenge is accepted at the AutoML-Conf 2022 and currently ongoing .",
      "arxiv_id": "2208.02821v1",
      "published": "2022-08-04",
      "categories": [
        "cs.LG",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2208.02821v1",
      "pdf_url": "https://arxiv.org/pdf/2208.02821v1.pdf"
    },
    {
      "title": "Integrating Symbolic Execution into the Fine-Tuning of Code-Generating LLMs",
      "authors": [
        "Marina Sakharova",
        "Abhinav Anand",
        "Mira Mezini"
      ],
      "summary": "Code-generating Large Language Models (LLMs) have become essential tools in modern software development, enhancing productivity and accelerating development. This paper aims to investigate the fine-tuning of code-generating LLMs using Reinforcement Learning and Direct Preference Optimization, further improving their performance. To achieve this, we enhance the training data for the reward model with the help of symbolic execution techniques, ensuring more comprehensive and objective data. With symbolic execution, we create a custom dataset that better captures the nuances in code evaluation. Our reward models, fine-tuned on this dataset, demonstrate significant improvements over the baseline, CodeRL, in estimating the quality of generated code. Our code-generating LLMs, trained with the help of reward model feedback, achieve similar results compared to the CodeRL benchmark.",
      "arxiv_id": "2504.15210v2",
      "published": "2025-04-21",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2504.15210v2",
      "pdf_url": "https://arxiv.org/pdf/2504.15210v2.pdf"
    },
    {
      "title": "ViSymRe: Vision-guided Multimodal Symbolic Regression",
      "authors": [
        "Da Li",
        "Junping Yin",
        "Jin Xu",
        "Xinxin Li",
        "Juan Zhang"
      ],
      "summary": "Extracting simple mathematical expression from an observational dataset to describe complex natural phenomena is one of the core objectives of artificial intelligence (AI). This field is known as symbolic regression (SR). Traditional SR models are based on genetic programming (GP) or reinforcement learning (RL), facing well-known challenges, such as low efficiency and overfitting. Recent studies have integrated SR with large language models (LLMs), enabling fast zero-shot inference by learning mappings from millions of dataset-expression pairs. However, since the input and output are inherently different modalities, such models often struggle to converge effectively. In this paper, we introduce ViSymRe, a vision-guided multimodal SR model that incorporates the third resource, expression graph, to bridge the modality gap. Different from traditional multimodal models, ViSymRe is trained to extract vision, termed virtual vision, from datasets, without relying on the global availability of expression graphs, which addresses the essential challenge of visual SR, i.e., expression graphs are not available during inference. Evaluation results on multiple mainstream benchmarks show that ViSymRe achieves more competitive performance than the state-of-the-art dataset-only baselines. The expressions predicted by ViSymRe not only fit the dataset well but are also simple and structurally accurate, goals that SR models strive to achieve.",
      "arxiv_id": "2412.11139v2",
      "published": "2024-12-15",
      "categories": [
        "cs.LG",
        "cs.AI",
        "cs.SC"
      ],
      "url": "https://arxiv.org/abs/2412.11139v2",
      "pdf_url": "https://arxiv.org/pdf/2412.11139v2.pdf"
    },
    {
      "title": "Language Models For Generalised PDDL Planning: Synthesising Sound and Programmatic Policies",
      "authors": [
        "Dillon Z. Chen",
        "Johannes Zenn",
        "Tristan Cinquin",
        "Sheila A. McIlraith"
      ],
      "summary": "We study the usage of language models (LMs) for planning over world models specified in the Planning Domain Definition Language (PDDL). We prompt LMs to generate Python programs that serve as generalised policies for solving PDDL problems from a given domain. Notably, our approach synthesises policies that are provably sound relative to the PDDL domain without reliance on external verifiers. We conduct experiments on competition benchmarks which show that our policies can solve more PDDL problems than PDDL planners and recent LM approaches within a fixed time and memory constraint. Our approach manifests in the LMPlan planner which can solve planning problems with several hundreds of relevant objects. Surprisingly, we observe that LMs used in our framework sometimes plan more effectively over PDDL problems written in meaningless symbols in place of natural language; e.g. rewriting (at dog kitchen) as (p2 o1 o3). This finding challenges hypotheses that LMs reason over word semantics and memorise solutions from its training corpus, and is worth further exploration.",
      "arxiv_id": "2508.18507v1",
      "published": "2025-08-25",
      "categories": [
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2508.18507v1",
      "pdf_url": "https://arxiv.org/pdf/2508.18507v1.pdf"
    },
    {
      "title": "Pac-Man Pete: An extensible framework for building AI in VEX Robotics",
      "authors": [
        "Jacob Zietek",
        "Nicholas Wade",
        "Cole Roberts",
        "Aref Malek",
        "Manish Pylla",
        "Will Xu",
        "Sagar Patil"
      ],
      "summary": "This technical report details VEX Robotics team BLRSAI's development of a fully autonomous robot for VEX Robotics' Tipping Point AI Competition. We identify and develop three separate critical components. This includes a Unity simulation and reinforcement learning model training pipeline, a malleable computer vision pipeline, and a data transfer pipeline to offload large computations from the VEX V5 Brain/micro-controller to an external computer. We give the community access to all of these components in hopes they can reuse and improve upon them in the future, and that it'll spark new ideas for autonomy as well as the necessary infrastructure and programs for AI in educational robotics.",
      "arxiv_id": "2211.14385v1",
      "published": "2022-11-25",
      "categories": [
        "cs.RO",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2211.14385v1",
      "pdf_url": "https://arxiv.org/pdf/2211.14385v1.pdf"
    },
    {
      "title": "ChatGPT for PLC/DCS Control Logic Generation",
      "authors": [
        "Heiko Koziolek",
        "Sten Gruener",
        "Virendra Ashiwal"
      ],
      "summary": "Large language models (LLMs) providing generative AI have become popular to support software engineers in creating, summarizing, optimizing, and documenting source code. It is still unknown how LLMs can support control engineers using typical control programming languages in programming tasks. Researchers have explored GitHub CoPilot or DeepMind AlphaCode for source code generation but did not yet tackle control logic programming. The contribution of this paper is an exploratory study, for which we created 100 LLM prompts in 10 representative categories to analyze control logic generation for of PLCs and DCS from natural language. We tested the prompts by generating answers with ChatGPT using the GPT-4 LLM. It generated syntactically correct IEC 61131-3 Structured Text code in many cases and demonstrated useful reasoning skills that could boost control engineer productivity. Our prompt collection is the basis for a more formal LLM benchmark to test and compare such models for control logic generation.",
      "arxiv_id": "2305.15809v1",
      "published": "2023-05-25",
      "categories": [
        "cs.SE",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2305.15809v1",
      "pdf_url": "https://arxiv.org/pdf/2305.15809v1.pdf"
    },
    {
      "title": "Optimizing AI-Assisted Code Generation",
      "authors": [
        "Simon Torka",
        "Sahin Albayrak"
      ],
      "summary": "In recent years, the rise of AI-assisted code-generation tools has significantly transformed software development. While code generators have mainly been used to support conventional software development, their use will be extended to powerful and secure AI systems. Systems capable of generating code, such as ChatGPT, OpenAI Codex, GitHub Copilot, and AlphaCode, take advantage of advances in machine learning (ML) and natural language processing (NLP) enabled by large language models (LLMs). However, it must be borne in mind that these models work probabilistically, which means that although they can generate complex code from natural language input, there is no guarantee for the functionality and security of the generated code.   However, to fully exploit the considerable potential of this technology, the security, reliability, functionality, and quality of the generated code must be guaranteed. This paper examines the implementation of these goals to date and explores strategies to optimize them. In addition, we explore how these systems can be optimized to create safe, high-performance, and executable artificial intelligence (AI) models, and consider how to improve their accessibility to make AI development more inclusive and equitable.",
      "arxiv_id": "2412.10953v1",
      "published": "2024-12-14",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2412.10953v1",
      "pdf_url": "https://arxiv.org/pdf/2412.10953v1.pdf"
    },
    {
      "title": "From Copilot to Pilot: Towards AI Supported Software Development",
      "authors": [
        "Rohith Pudari",
        "Neil A. Ernst"
      ],
      "summary": "AI-supported programming has arrived, as shown by the introduction and successes of large language models for code, such as Copilot/Codex (Github/OpenAI) and AlphaCode (DeepMind). Above human average performance on programming challenges is now possible. However, software engineering is much more than solving programming contests. Moving beyond code completion to AI-supported software engineering will require an AI system that can, among other things, understand how to avoid code smells, to follow language idioms, and eventually (maybe!) propose rational software designs. In this study, we explore the current limitations of AI-supported code completion tools like Copilot and offer a simple taxonomy for understanding the classification of AI-supported code completion tools in this space. We first perform an exploratory study on Copilot's code suggestions for language idioms and code smells. Copilot does not follow language idioms and avoid code smells in most of our test scenarios. We then conduct additional investigation to determine the current boundaries of AI-supported code completion tools like Copilot by introducing a taxonomy of software abstraction hierarchies where 'basic programming functionality' such as code compilation and syntax checking is at the least abstract level, software architecture analysis and design are at the most abstract level. We conclude by providing a discussion on challenges for future development of AI-supported code completion tools to reach the design level of abstraction in our taxonomy.",
      "arxiv_id": "2303.04142v1",
      "published": "2023-03-07",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2303.04142v1",
      "pdf_url": "https://arxiv.org/pdf/2303.04142v1.pdf"
    },
    {
      "title": "Agent-RLVR: Training Software Engineering Agents via Guidance and Environment Rewards",
      "authors": [
        "Jeff Da",
        "Clinton Wang",
        "Xiang Deng",
        "Yuntao Ma",
        "Nikhil Barhate",
        "Sean Hendryx"
      ],
      "summary": "Reinforcement Learning from Verifiable Rewards (RLVR) has been widely adopted as the de facto method for enhancing the reasoning capabilities of large language models and has demonstrated notable success in verifiable domains like math and competitive programming tasks. However, the efficacy of RLVR diminishes significantly when applied to agentic environments. These settings, characterized by multi-step, complex problem solving, lead to high failure rates even for frontier LLMs, as the reward landscape is too sparse for effective model training via conventional RLVR. In this work, we introduce Agent-RLVR, a framework that makes RLVR effective in challenging agentic settings, with an initial focus on software engineering tasks. Inspired by human pedagogy, Agent-RLVR introduces agent guidance, a mechanism that actively steers the agent towards successful trajectories by leveraging diverse informational cues. These cues, ranging from high-level strategic plans to dynamic feedback on the agent's errors and environmental interactions, emulate a teacher's guidance, enabling the agent to navigate difficult solution spaces and promotes active self-improvement via additional environment exploration. In the Agent-RLVR training loop, agents first attempt to solve tasks to produce initial trajectories, which are then validated by unit tests and supplemented with agent guidance. Agents then reattempt with guidance, and the agent policy is updated with RLVR based on the rewards of these guided trajectories. Agent-RLVR elevates the pass@1 performance of Qwen-2.5-72B-Instruct from 9.4% to 22.4% on SWE-Bench Verified. We find that our guidance-augmented RLVR data is additionally useful for test-time reward model training, shown by further boosting pass@1 to 27.8%. Agent-RLVR lays the groundwork for training agents with RLVR in complex, real-world environments where conventional RL methods struggle.",
      "arxiv_id": "2506.11425v2",
      "published": "2025-06-13",
      "categories": [
        "cs.CL",
        "cs.AI"
      ],
      "url": "https://arxiv.org/abs/2506.11425v2",
      "pdf_url": "https://arxiv.org/pdf/2506.11425v2.pdf"
    },
    {
      "title": "Natural Language Generation and Understanding of Big Code for AI-Assisted Programming: A Review",
      "authors": [
        "Man Fai Wong",
        "Shangxin Guo",
        "Ching Nam Hang",
        "Siu Wai Ho",
        "Chee Wei Tan"
      ],
      "summary": "This paper provides a comprehensive review of the literature concerning the utilization of Natural Language Processing (NLP) techniques, with a particular focus on transformer-based large language models (LLMs) trained using Big Code, within the domain of AI-assisted programming tasks. LLMs, augmented with software naturalness, have played a crucial role in facilitating AI-assisted programming applications, including code generation, code completion, code translation, code refinement, code summarization, defect detection, and clone detection. Notable examples of such applications include the GitHub Copilot powered by OpenAI's Codex and DeepMind AlphaCode. This paper presents an overview of the major LLMs and their applications in downstream tasks related to AI-assisted programming. Furthermore, it explores the challenges and opportunities associated with incorporating NLP techniques with software naturalness in these applications, with a discussion on extending AI-assisted programming capabilities to Apple's Xcode for mobile software development. This paper also presents the challenges of and opportunities for incorporating NLP techniques with software naturalness, empowering developers with advanced coding assistance and streamlining the software development process.",
      "arxiv_id": "2307.02503v1",
      "published": "2023-07-04",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.CL"
      ],
      "url": "https://arxiv.org/abs/2307.02503v1",
      "pdf_url": "https://arxiv.org/pdf/2307.02503v1.pdf"
    },
    {
      "title": "What is it like to program with artificial intelligence?",
      "authors": [
        "Advait Sarkar",
        "Andrew D. Gordon",
        "Carina Negreanu",
        "Christian Poelitz",
        "Sruti Srinivasa Ragavan",
        "Ben Zorn"
      ],
      "summary": "Large language models, such as OpenAI's codex and Deepmind's AlphaCode, can generate code to solve a variety of problems expressed in natural language. This technology has already been commercialised in at least one widely-used programming editor extension: GitHub Copilot.   In this paper, we explore how programming with large language models (LLM-assisted programming) is similar to, and differs from, prior conceptualisations of programmer assistance. We draw upon publicly available experience reports of LLM-assisted programming, as well as prior usability and design studies. We find that while LLM-assisted programming shares some properties of compilation, pair programming, and programming via search and reuse, there are fundamental differences both in the technical possibilities as well as the practical experience. Thus, LLM-assisted programming ought to be viewed as a new way of programming with its own distinct properties and challenges.   Finally, we draw upon observations from a user study in which non-expert end user programmers use LLM-assisted tools for solving data tasks in spreadsheets. We discuss the issues that might arise, and open research challenges, in applying large language models to end-user programming, particularly with users who have little or no programming expertise.",
      "arxiv_id": "2208.06213v2",
      "published": "2022-08-12",
      "categories": [
        "cs.HC",
        "cs.AI",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2208.06213v2",
      "pdf_url": "https://arxiv.org/pdf/2208.06213v2.pdf"
    },
    {
      "title": "Follow-up Attention: An Empirical Study of Developer and Neural Model Code Exploration",
      "authors": [
        "Matteo Paltenghi",
        "Rahul Pandita",
        "Austin Z. Henley",
        "Albert Ziegler"
      ],
      "summary": "Recent neural models of code, such as OpenAI Codex and AlphaCode, have demonstrated remarkable proficiency at code generation due to the underlying attention mechanism. However, it often remains unclear how the models actually process code, and to what extent their reasoning and the way their attention mechanism scans the code matches the patterns of developers. A poor understanding of the model reasoning process limits the way in which current neural models are leveraged today, so far mostly for their raw prediction. To fill this gap, this work studies how the processed attention signal of three open large language models - CodeGen, InCoder and GPT-J - agrees with how developers look at and explore code when each answers the same sensemaking questions about code. Furthermore, we contribute an open-source eye-tracking dataset comprising 92 manually-labeled sessions from 25 developers engaged in sensemaking tasks. We empirically evaluate five heuristics that do not use the attention and ten attention-based post-processing approaches of the attention signal of CodeGen against our ground truth of developers exploring code, including the novel concept of follow-up attention which exhibits the highest agreement between model and human attention. Our follow-up attention method can predict the next line a developer will look at with 47% accuracy. This outperforms the baseline prediction accuracy of 42.3%, which uses the session history of other developers to recommend the next line. These results demonstrate the potential of leveraging the attention signal of pre-trained models for effective code exploration.",
      "arxiv_id": "2210.05506v2",
      "published": "2022-10-11",
      "categories": [
        "cs.SE",
        "cs.AI",
        "cs.HC",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2210.05506v2",
      "pdf_url": "https://arxiv.org/pdf/2210.05506v2.pdf"
    },
    {
      "title": "AlphaDou: High-Performance End-to-End Doudizhu AI Integrating Bidding",
      "authors": [
        "Chang Lei",
        "Huan Lei"
      ],
      "summary": "Artificial intelligence for card games has long been a popular topic in AI research. In recent years, complex card games like Mahjong and Texas Hold'em have been solved, with corresponding AI programs reaching the level of human experts. However, the game of Doudizhu presents significant challenges due to its vast state/action space and unique characteristics involving reasoning about competition and cooperation, making the game extremely difficult to solve.The RL model Douzero, trained using the Deep Monte Carlo algorithm framework, has shown excellent performance in Doudizhu. However, there are differences between its simplified game environment and the actual Doudizhu environment, and its performance is still a considerable distance from that of human experts. This paper modifies the Deep Monte Carlo algorithm framework by using reinforcement learning to obtain a neural network that simultaneously estimates win rates and expectations. The action space is pruned using expectations, and strategies are generated based on win rates. The modified algorithm enables the AI to perform the full range of tasks in the Doudizhu game, including bidding and cardplay. The model was trained in a actual Doudizhu environment and achieved state-of-the-art performance among publicly available models. We hope that this new framework will provide valuable insights for AI development in other bidding-based games.",
      "arxiv_id": "2407.10279v2",
      "published": "2024-07-14",
      "categories": [
        "cs.AI",
        "cs.GT",
        "cs.MA"
      ],
      "url": "https://arxiv.org/abs/2407.10279v2",
      "pdf_url": "https://arxiv.org/pdf/2407.10279v2.pdf"
    },
    {
      "title": "Parsel: Algorithmic Reasoning with Language Models by Composing Decompositions",
      "authors": [
        "Eric Zelikman",
        "Qian Huang",
        "Gabriel Poesia",
        "Noah D. Goodman",
        "Nick Haber"
      ],
      "summary": "Despite recent success in large language model (LLM) reasoning, LLMs struggle with hierarchical multi-step reasoning tasks like generating complex programs. For these tasks, humans often start with a high-level algorithmic design and implement each part gradually. We introduce Parsel, a framework enabling automatic implementation and validation of complex algorithms with code LLMs. With Parsel, we automatically decompose algorithmic tasks into hierarchical natural language function descriptions and then search over combinations of possible function implementations using tests. We show that Parsel can be used across domains requiring hierarchical reasoning, including program synthesis and robotic planning. We find that, using Parsel, LLMs solve more competition-level problems in the APPS dataset, resulting in pass rates over 75\\% higher than prior results from directly sampling AlphaCode and Codex, while often using a smaller sample budget. Moreover, with automatically generated tests, we find that Parsel can improve the state-of-the-art pass@1 performance on HumanEval from 67\\% to 85\\%. We also find that LLM-generated robotic plans using Parsel are more than twice as likely to be considered accurate than directly generated plans. Lastly, we explore how Parsel addresses LLM limitations and discuss how Parsel may be useful for human programmers. We release our code at https://github.com/ezelikman/parsel",
      "arxiv_id": "2212.10561v3",
      "published": "2022-12-20",
      "categories": [
        "cs.CL",
        "cs.AI",
        "cs.LG"
      ],
      "url": "https://arxiv.org/abs/2212.10561v3",
      "pdf_url": "https://arxiv.org/pdf/2212.10561v3.pdf"
    },
    {
      "title": "Code Detection for Hardware Acceleration Using Large Language Models",
      "authors": [
        "Pablo Antonio Mart\u00ednez",
        "Gregorio Bernab\u00e9",
        "Jos\u00e9 Manuel Garc\u00eda"
      ],
      "summary": "Large language models (LLMs) have been massively applied to many tasks, often surpassing state-of-the-art approaches. While their effectiveness in code generation has been extensively studied (e.g., AlphaCode), their potential for code detection remains unexplored.   This work presents the first analysis of code detection using LLMs. Our study examines essential kernels, including matrix multiplication, convolution, and fast-fourier transform, implemented in C/C++. We propose both a preliminary, naive prompt and a novel prompting strategy for code detection.   Results reveal that conventional prompting achieves great precision but poor accuracy (68.8%, 22.3%, and 79.2% for GEMM, convolution, and FFT, respectively) due to a high number of false positives. Our novel prompting strategy substantially reduces false positives, resulting in excellent overall accuracy (91.1%, 97.9%, and 99.7%, respectively). These results pose a considerable challenge to existing state-of-the-art code detection methods.",
      "arxiv_id": "2307.10348v1",
      "published": "2023-07-19",
      "categories": [
        "cs.SE",
        "cs.LG",
        "cs.PL"
      ],
      "url": "https://arxiv.org/abs/2307.10348v1",
      "pdf_url": "https://arxiv.org/pdf/2307.10348v1.pdf"
    },
    {
      "title": "LLM-Assisted Code Cleaning For Training Accurate Code Generators",
      "authors": [
        "Naman Jain",
        "Tianjun Zhang",
        "Wei-Lin Chiang",
        "Joseph E. Gonzalez",
        "Koushik Sen",
        "Ion Stoica"
      ],
      "summary": "Natural language to code generation is an important application area of LLMs and has received wide attention from the community. The majority of relevant studies have exclusively concentrated on increasing the quantity and functional correctness of training sets while disregarding other stylistic elements of programs. More recently, data quality has garnered a lot of interest and multiple works have showcased its importance for improving performance. In this work, we investigate data quality for code and find that making the code more structured and readable leads to improved code generation performance of the system. We build a novel data-cleaning pipeline that uses these principles to transform existing programs by 1.) renaming variables, 2.) modularizing and decomposing complex code into smaller helper sub-functions, and 3.) inserting natural-language based plans via LLM based transformations. We evaluate our approach on two challenging algorithmic code generation benchmarks and find that fine-tuning CodeLLaMa-7B on our transformed modularized programs improves the performance by up to 30% compared to fine-tuning on the original dataset. Additionally, we demonstrate improved performance from using a smaller amount of higher-quality data, finding that a model fine-tuned on the entire original dataset is outperformed by a model trained on 15% of our cleaned dataset. Even in comparison to closed-source models, our models outperform the much larger AlphaCoder models.",
      "arxiv_id": "2311.14904v1",
      "published": "2023-11-25",
      "categories": [
        "cs.LG",
        "cs.SE"
      ],
      "url": "https://arxiv.org/abs/2311.14904v1",
      "pdf_url": "https://arxiv.org/pdf/2311.14904v1.pdf"
    }
  ]
}