<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-03902423</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-22T15:10:48+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">EAGER: Asking and Answering Questions for Automatic Reward Shaping in Language-guided RL</title>
            <author role="aut">
              <persName>
                <forename type="first">Thomas</forename>
                <surname>Carta</surname>
              </persName>
              <email type="md5">ee2fcc2ebc1f69d1d251439950ba6768</email>
              <email type="domain">polytechnique.edu</email>
              <idno type="idhal" notation="string">thomas-carta</idno>
              <idno type="idhal" notation="numeric">750836</idno>
              <idno type="halauthorid" notation="string">55287-750836</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-3145-549X</idno>
              <affiliation ref="#struct-60156"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sylvain</forename>
                <surname>Lamprier</surname>
              </persName>
              <email type="md5">942471cad263f780de2404e093709852</email>
              <email type="domain">yahoo.fr</email>
              <idno type="idhal" notation="string">sylvain-lamprier</idno>
              <idno type="idhal" notation="numeric">740402</idno>
              <idno type="halauthorid" notation="string">30958-740402</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-2508-922X</idno>
              <idno type="IDREF">https://www.idref.fr/142632201</idno>
              <idno type="ISNI">http://isni.org/isni/0000000140859141</idno>
              <idno type="VIAF">https://viaf.org/viaf/200144138</idno>
              <idno type="RESEARCHERID">http://www.researcherid.com/rid/DYO-1764-2022</idno>
              <affiliation ref="#struct-6258"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Pierre-Yves</forename>
                <surname>Oudeyer</surname>
              </persName>
              <email type="md5">9ff76693e026f7a72312fc2b99729d8a</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">pyoudeyer</idno>
              <idno type="idhal" notation="numeric">6675</idno>
              <idno type="halauthorid" notation="string">2543221-6675</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-9404-7613</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-1277-130X</idno>
              <idno type="IDREF">https://www.idref.fr/081674481</idno>
              <affiliation ref="#struct-60156"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Olivier</forename>
                <surname>Sigaud</surname>
              </persName>
              <email type="md5">1a3a7a628849e00426cd103ac2d585cb</email>
              <email type="domain">upmc.fr</email>
              <idno type="idhal" notation="string">olivier-sigaud</idno>
              <idno type="idhal" notation="numeric">14932</idno>
              <idno type="halauthorid" notation="string">15986-14932</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-8544-0229</idno>
              <idno type="IDREF">https://www.idref.fr/072724714</idno>
              <idno type="ARXIV">https://arxiv.org/a/sigaud_o_1</idno>
              <idno type="ISNI">http://isni.org/isni/0000000073281983</idno>
              <idno type="VIAF">https://viaf.org/viaf/43669783</idno>
              <idno type="RESEARCHERID">http://www.researcherid.com/rid/FWG-2898-2022</idno>
              <affiliation ref="#struct-541937"/>
              <affiliation ref="#struct-60156"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Thomas</forename>
                <surname>Carta</surname>
              </persName>
              <email type="md5">5fec74876d86ce61e52f6918e115b2ae</email>
              <email type="domain">hotmail.fr</email>
            </editor>
            <funder ref="#projanr-53270"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2022-12-15 22:01:58</date>
              <date type="whenModified">2025-08-20 10:52:44</date>
              <date type="whenReleased">2022-12-16 09:18:32</date>
              <date type="whenProduced">2022-11-28</date>
              <date type="whenEndEmbargoed">2022-12-15</date>
              <ref type="file" target="https://inria.hal.science/hal-03902423v1/document">
                <date notBefore="2022-12-15"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://inria.hal.science/hal-03902423v1/file/EAGER__Asking_and_Answering_Questions_for_Automatic_Reward_Shaping_in_Language_guided_RL-6.pdf" id="file-3902423-3412969">
                <date notBefore="2022-12-15"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1108361">
                <persName>
                  <forename>Thomas</forename>
                  <surname>Carta</surname>
                </persName>
                <email type="md5">5fec74876d86ce61e52f6918e115b2ae</email>
                <email type="domain">hotmail.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-03902423</idno>
            <idno type="halUri">https://inria.hal.science/hal-03902423</idno>
            <idno type="halBibtex">carta:hal-03902423</idno>
            <idno type="halRefHtml">&lt;i&gt;NeurIPS 2022 - Thirty-sixth Conference on Neural Information Processing Systems&lt;/i&gt;, Nov 2022, Nouvelle-Orléans, United States</idno>
            <idno type="halRef">NeurIPS 2022 - Thirty-sixth Conference on Neural Information Processing Systems, Nov 2022, Nouvelle-Orléans, United States</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-3902423-3412969"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="UNIV-ANGERS">Université d'Angers</idno>
            <idno type="stamp" n="INRIA-BORDEAUX">INRIA Bordeaux - Sud-Ouest</idno>
            <idno type="stamp" n="INRIA_TEST">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="ISIR" corresp="SORBONNE-UNIVERSITE">Institut des Systèmes Intelligents et de Robotique</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="GENCI">Publications ayant eu recours aux supercalculateurs du GENCI</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="IP_PARIS">Institut Polytechnique de Paris</idno>
            <idno type="stamp" n="LERIA" corresp="UNIV-ANGERS">Laboratoire d'Etude et de Recherche en Informatique d'Angers</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="ISIR_AMAC" corresp="ISIR">Architectures et Modèles de l'Adaptation et de la Cognition</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="ISIR_MLIA" corresp="ISIR">Machine Learning and Information Access</idno>
            <idno type="stamp" n="SUPRA_INGENIERIE">Ingénierie</idno>
            <idno type="stamp" n="ANR-IA-19" corresp="ANR-IA">ANR-IA-19</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
            <idno type="stamp" n="ENSTA-PARIS">ENSTA-PARIS</idno>
            <idno type="stamp" n="ENSTA">Ensta</idno>
            <idno type="stamp" n="DEPARTEMENT-DE-MATHEMATIQUES">Collection du Département de Mathématiques</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">EAGER: Asking and Answering Questions for Automatic Reward Shaping in Language-guided RL</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Thomas</forename>
                    <surname>Carta</surname>
                  </persName>
                  <email type="md5">ee2fcc2ebc1f69d1d251439950ba6768</email>
                  <email type="domain">polytechnique.edu</email>
                  <idno type="idhal" notation="string">thomas-carta</idno>
                  <idno type="idhal" notation="numeric">750836</idno>
                  <idno type="halauthorid" notation="string">55287-750836</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-3145-549X</idno>
                  <affiliation ref="#struct-60156"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sylvain</forename>
                    <surname>Lamprier</surname>
                  </persName>
                  <email type="md5">942471cad263f780de2404e093709852</email>
                  <email type="domain">yahoo.fr</email>
                  <idno type="idhal" notation="string">sylvain-lamprier</idno>
                  <idno type="idhal" notation="numeric">740402</idno>
                  <idno type="halauthorid" notation="string">30958-740402</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-2508-922X</idno>
                  <idno type="IDREF">https://www.idref.fr/142632201</idno>
                  <idno type="ISNI">http://isni.org/isni/0000000140859141</idno>
                  <idno type="VIAF">https://viaf.org/viaf/200144138</idno>
                  <idno type="RESEARCHERID">http://www.researcherid.com/rid/DYO-1764-2022</idno>
                  <affiliation ref="#struct-6258"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Pierre-Yves</forename>
                    <surname>Oudeyer</surname>
                  </persName>
                  <email type="md5">9ff76693e026f7a72312fc2b99729d8a</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">pyoudeyer</idno>
                  <idno type="idhal" notation="numeric">6675</idno>
                  <idno type="halauthorid" notation="string">2543221-6675</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-9404-7613</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-1277-130X</idno>
                  <idno type="IDREF">https://www.idref.fr/081674481</idno>
                  <affiliation ref="#struct-60156"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Olivier</forename>
                    <surname>Sigaud</surname>
                  </persName>
                  <email type="md5">1a3a7a628849e00426cd103ac2d585cb</email>
                  <email type="domain">upmc.fr</email>
                  <idno type="idhal" notation="string">olivier-sigaud</idno>
                  <idno type="idhal" notation="numeric">14932</idno>
                  <idno type="halauthorid" notation="string">15986-14932</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-8544-0229</idno>
                  <idno type="IDREF">https://www.idref.fr/072724714</idno>
                  <idno type="ARXIV">https://arxiv.org/a/sigaud_o_1</idno>
                  <idno type="ISNI">http://isni.org/isni/0000000073281983</idno>
                  <idno type="VIAF">https://viaf.org/viaf/43669783</idno>
                  <idno type="RESEARCHERID">http://www.researcherid.com/rid/FWG-2898-2022</idno>
                  <affiliation ref="#struct-541937"/>
                  <affiliation ref="#struct-60156"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>NeurIPS 2022 - Thirty-sixth Conference on Neural Information Processing Systems</title>
                  <date type="start">2022-11-28</date>
                  <date type="end">2022-12-09</date>
                  <settlement>Nouvelle-Orléans</settlement>
                  <country key="US">United States</country>
                </meeting>
                <imprint/>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="math">Mathematics [math]</classCode>
              <classCode scheme="halDomain" n="stat">Statistics [stat]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Reinforcement learning (RL) in long horizon and sparse reward tasks is notoriously difficult and requires a lot of training steps. A standard solution to speed up the process is to leverage additional reward signals, shaping it to better guide the learning process. In the context of language-conditioned RL, the abstraction and generalisation properties of the language input provide opportunities for more efficient ways of shaping the reward. In this paper, we leverage this idea and propose an automated reward shaping method where the agent extracts auxiliary objectives from the general language goal. These auxiliary objectives use a question generation (QG) and question answering (QA) system: they consist of questions leading the agent to try to reconstruct partial information about the global goal using its own trajectory. When it succeeds, it receives an intrinsic reward proportional to its confidence in its answer. This incentivizes the agent to generate trajectories which unambiguously explain various aspects of the general language goal. Our experimental study shows that this approach, which does not require engineer intervention to design the auxiliary objectives, improves sample efficiency by effectively directing exploration.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-60156" status="OLD">
          <idno type="RNSR">200820949R</idno>
          <orgName>Flowing Epigenetic Robots and Systems</orgName>
          <orgName type="acronym">Flowers</orgName>
          <date type="start">2011-01-01</date>
          <date type="end">2025-02-28</date>
          <desc>
            <address>
              <addrLine>200, avenue de la Vieille Tour 33405 Talence cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://flowers.inria.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-104751" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation active="#struct-234188" type="direct"/>
            <relation active="#struct-300065" type="indirect"/>
            <relation active="#struct-563936" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-6258" status="VALID">
          <idno type="IdRef">06870397X</idno>
          <idno type="RNSR">199614159M</idno>
          <orgName>Laboratoire d'Etudes et de Recherche en Informatique d'Angers</orgName>
          <orgName type="acronym">LERIA</orgName>
          <date type="start">1996-01-01</date>
          <desc>
            <address>
              <addrLine>Université d'Angers, LERIA - Facultés des Sciences, 2 Boulevard Lavoisier, Bâtiments G et H - 49045 Angers CEDEX 01</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://leria.univ-angers.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-74911" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-541937" status="VALID">
          <idno type="IdRef">241179122</idno>
          <idno type="RNSR">200918463J</idno>
          <idno type="ROR">05neq8668</idno>
          <orgName>Institut des Systèmes Intelligents et de Robotique</orgName>
          <orgName type="acronym">ISIR</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>Sorbonne-Université, Boite courrier 173 4 Place JUSSIEU 75252 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.isir.upmc.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413221" type="direct"/>
            <relation name="UMR7222" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-104751" status="VALID">
          <idno type="RNSR">200818243Z</idno>
          <idno type="ROR">https://ror.org/03tjcj052</idno>
          <orgName>Centre Inria de l'Université de Bordeaux</orgName>
          <desc>
            <address>
              <addrLine>200, avenue de la Vieille Tour, 33405 Talence</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/bordeaux/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="regrouplaboratory" xml:id="struct-234188" status="OLD">
          <idno type="IdRef">203564529</idno>
          <idno type="RNSR">200920620D</idno>
          <orgName>Unité d'Informatique et d'Ingénierie des Systèmes</orgName>
          <orgName type="acronym">U2IS</orgName>
          <date type="end">2024-12-31</date>
          <desc>
            <address>
              <addrLine>828, boulevard des Maréchaux, 91762 Palaiseau CEDEX</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://u2is.ensta-paris.fr/index.php?lang=fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300065" type="direct"/>
            <relation active="#struct-563936" type="indirect"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300065" status="OLD">
          <idno type="IdRef">028139577</idno>
          <idno type="ISNI">0000000122070120</idno>
          <idno type="ROR">https://ror.org/0309cs235</idno>
          <idno type="Wikidata">Q838454</idno>
          <orgName>École Nationale Supérieure de Techniques Avancées</orgName>
          <orgName type="acronym">ENSTA Paris</orgName>
          <date type="end">2024-12-31</date>
          <desc>
            <address>
              <addrLine>828, boulevard des Maréchaux 91120 Palaiseau</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ensta-paris.fr/fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-563936" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-563936" status="VALID">
          <idno type="IdRef">238327159</idno>
          <idno type="ISNI">0000000502717600</idno>
          <idno type="ROR">https://ror.org/042tfbd02</idno>
          <idno type="Wikidata">Q48759778</idno>
          <orgName>Institut Polytechnique de Paris</orgName>
          <orgName type="acronym">IP Paris</orgName>
          <date type="start">2019-06-02</date>
          <desc>
            <address>
              <addrLine>Route de Saclay, 91120 Palaiseau Cedex, France</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ip-paris.fr</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-74911" status="VALID">
          <idno type="IdRef">026402920</idno>
          <idno type="ISNI">0000000122483363</idno>
          <idno type="ROR">https://ror.org/04yrqp957</idno>
          <orgName>Université d'Angers</orgName>
          <orgName type="acronym">UA</orgName>
          <date type="start">1971-10-25</date>
          <desc>
            <address>
              <addrLine>Université d'Angers - 40 Rue de Rennes, BP 73532 - 49035 Angers CEDEX 01</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-angers.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-413221" status="VALID">
          <idno type="IdRef">221333754</idno>
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Sorbonne Université</orgName>
          <orgName type="acronym">SU</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>21 rue de l’École de médecine - 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sorbonne-universite.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-53270" status="VALID">
          <idno type="anr">ANR-19-CHIA-0004</idno>
          <orgName>DeepCuriosity</orgName>
          <desc>DeepCuriosity: Exploration dirigée par la curiosité et apprentissage développemental en IA, et applications pour les agents autonomes, la découverte automatique et les technologies de l'éducation.</desc>
          <date type="start">2019</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>