rl-taxonomy.gv.svg

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
 "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.38.0 (20140413.2041)
 -->
<!-- Title: %3 Pages: 1 -->
<svg width="1581pt" height="1652pt"
 viewBox="0.00 0.00 1581.00 1652.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 1648)">
<title>%3</title>
<polygon fill="white" stroke="none" points="-4,4 -4,-1648 1577,-1648 1577,4 -4,4"/>
<g id="clust1" class="cluster"><title>clusterTimeline</title>
<polygon fill="#707070" stroke="#707070" stroke-width="2" points="44,-8 44,-24 1544,-24 1544,-8 44,-8"/>
</g>
<g id="clust2" class="cluster"><title>clusterModel Free</title>
<path fill="#f7fdff" stroke="black" d="M188.5,-692C188.5,-692 1553,-692 1553,-692 1559,-692 1565,-698 1565,-704 1565,-704 1565,-1624 1565,-1624 1565,-1630 1559,-1636 1553,-1636 1553,-1636 188.5,-1636 188.5,-1636 182.5,-1636 176.5,-1630 176.5,-1624 176.5,-1624 176.5,-704 176.5,-704 176.5,-698 182.5,-692 188.5,-692"/>
<text text-anchor="middle" x="870.75" y="-1619.2" font-family="arial black" font-size="16.00">Model Free</text>
</g>
<g id="clust3" class="cluster"><title>clusterValue Gradient</title>
<path fill="#daf0f6" stroke="black" stroke-dasharray="5,2" d="M312.5,-1302C312.5,-1302 1545,-1302 1545,-1302 1551,-1302 1557,-1308 1557,-1314 1557,-1314 1557,-1590 1557,-1590 1557,-1596 1551,-1602 1545,-1602 1545,-1602 312.5,-1602 312.5,-1602 306.5,-1602 300.5,-1596 300.5,-1590 300.5,-1590 300.5,-1314 300.5,-1314 300.5,-1308 306.5,-1302 312.5,-1302"/>
<text text-anchor="middle" x="928.75" y="-1585.2" font-family="arial black" font-size="16.00">Value Gradient</text>
</g>
<g id="clust4" class="cluster"><title>clusterPolicy Gradient/Actor&#45;Critic</title>
<path fill="#daf0f6" stroke="black" stroke-dasharray="5,2" d="M311,-700C311,-700 1344.5,-700 1344.5,-700 1350.5,-700 1356.5,-706 1356.5,-712 1356.5,-712 1356.5,-1282 1356.5,-1282 1356.5,-1288 1350.5,-1294 1344.5,-1294 1344.5,-1294 311,-1294 311,-1294 305,-1294 299,-1288 299,-1282 299,-1282 299,-712 299,-712 299,-706 305,-700 311,-700"/>
<text text-anchor="middle" x="827.75" y="-1277.2" font-family="arial black" font-size="16.00">Policy Gradient/Actor&#45;Critic</text>
</g>
<g id="clust5" class="cluster"><title>clusterModel Based</title>
<path fill="#dafdda" stroke="black" d="M183,-234C183,-234 1444,-234 1444,-234 1450,-234 1456,-240 1456,-246 1456,-246 1456,-672 1456,-672 1456,-678 1450,-684 1444,-684 1444,-684 183,-684 183,-684 177,-684 171,-678 171,-672 171,-672 171,-246 171,-246 171,-240 177,-234 183,-234"/>
<text text-anchor="middle" x="813.5" y="-667.2" font-family="arial black" font-size="16.00">Model Based</text>
</g>
<g id="clust6" class="cluster"><title>clusterMeta&#45;RL</title>
<path fill="#f5f5da" stroke="black" d="M624,-34C624,-34 1326,-34 1326,-34 1332,-34 1338,-40 1338,-46 1338,-46 1338,-214 1338,-214 1338,-220 1332,-226 1326,-226 1326,-226 624,-226 624,-226 618,-226 612,-220 612,-214 612,-214 612,-46 612,-46 612,-40 618,-34 624,-34"/>
<text text-anchor="middle" x="975" y="-209.2" font-family="arial black" font-size="16.00">Meta&#45;RL</text>
</g>
<!-- 1950s -->
<g id="node1" class="node"><title>1950s</title>
<text text-anchor="middle" x="71" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">1950s</text>
</g>
<!-- 1980&#45;90s -->
<g id="node2" class="node"><title>1980&#45;90s</title>
<text text-anchor="middle" x="515.5" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">1980&#45;90s</text>
</g>
<!-- 1950s&#45;&gt;1980&#45;90s -->
<g id="edge1" class="edge"><title>1950s&#45;&gt;1980&#45;90s</title>
<path fill="none" stroke="white" d="M98.0284,-16C172.072,-16 384.298,-16 473.852,-16"/>
<polygon fill="white" stroke="white" points="473.897,-19.5001 483.897,-16 473.897,-12.5001 473.897,-19.5001"/>
</g>
<!-- 2000s -->
<g id="node3" class="node"><title>2000s</title>
<text text-anchor="middle" x="652" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2000s</text>
</g>
<!-- 1980&#45;90s&#45;&gt;2000s -->
<g id="edge2" class="edge"><title>1980&#45;90s&#45;&gt;2000s</title>
<path fill="none" stroke="white" d="M547.02,-16C567.044,-16 593.335,-16 614.47,-16"/>
<polygon fill="white" stroke="white" points="614.685,-19.5001 624.685,-16 614.685,-12.5001 614.685,-19.5001"/>
</g>
<!-- 2010&#45;2015 -->
<g id="node4" class="node"><title>2010&#45;2015</title>
<text text-anchor="middle" x="816.5" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2010&#45;2015</text>
</g>
<!-- 2000s&#45;&gt;2010&#45;2015 -->
<g id="edge3" class="edge"><title>2000s&#45;&gt;2010&#45;2015</title>
<path fill="none" stroke="white" d="M679.108,-16C703.827,-16 741.437,-16 770.907,-16"/>
<polygon fill="white" stroke="white" points="770.957,-19.5001 780.957,-16 770.957,-12.5001 770.957,-19.5001"/>
</g>
<!-- 2016 -->
<g id="node5" class="node"><title>2016</title>
<text text-anchor="middle" x="936.5" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2016</text>
</g>
<!-- 2010&#45;2015&#45;&gt;2016 -->
<g id="edge4" class="edge"><title>2010&#45;2015&#45;&gt;2016</title>
<path fill="none" stroke="white" d="M852.266,-16C866.927,-16 884.004,-16 898.861,-16"/>
<polygon fill="white" stroke="white" points="899.22,-19.5001 909.22,-16 899.22,-12.5001 899.22,-19.5001"/>
</g>
<!-- 2017 -->
<g id="node6" class="node"><title>2017</title>
<text text-anchor="middle" x="1147.5" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2017</text>
</g>
<!-- 2016&#45;&gt;2017 -->
<g id="edge5" class="edge"><title>2016&#45;&gt;2017</title>
<path fill="none" stroke="white" d="M963.547,-16C1000.57,-16 1068.55,-16 1110.41,-16"/>
<polygon fill="white" stroke="white" points="1110.45,-19.5001 1120.45,-16 1110.45,-12.5001 1110.45,-19.5001"/>
</g>
<!-- 2018 -->
<g id="node7" class="node"><title>2018</title>
<text text-anchor="middle" x="1302.5" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2018</text>
</g>
<!-- 2017&#45;&gt;2018 -->
<g id="edge6" class="edge"><title>2017&#45;&gt;2018</title>
<path fill="none" stroke="white" d="M1174.8,-16C1199.73,-16 1237.38,-16 1265.17,-16"/>
<polygon fill="white" stroke="white" points="1265.29,-19.5001 1275.29,-16 1265.29,-12.5001 1265.29,-19.5001"/>
</g>
<!-- 2019 -->
<g id="node8" class="node"><title>2019</title>
<text text-anchor="middle" x="1417.5" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2019</text>
</g>
<!-- 2018&#45;&gt;2019 -->
<g id="edge7" class="edge"><title>2018&#45;&gt;2019</title>
<path fill="none" stroke="white" d="M1329.71,-16C1344.68,-16 1363.69,-16 1380.1,-16"/>
<polygon fill="white" stroke="white" points="1380.43,-19.5001 1390.43,-16 1380.43,-12.5001 1380.43,-19.5001"/>
</g>
<!-- 2020 -->
<g id="node9" class="node"><title>2020</title>
<text text-anchor="middle" x="1517" y="-12.3" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="14.00" fill="white">2020</text>
</g>
<!-- 2019&#45;&gt;2020 -->
<g id="edge8" class="edge"><title>2019&#45;&gt;2020</title>
<path fill="none" stroke="white" d="M1444.63,-16C1455.34,-16 1467.92,-16 1479.51,-16"/>
<polygon fill="white" stroke="white" points="1479.8,-19.5001 1489.8,-16 1479.8,-12.5001 1479.8,-19.5001"/>
</g>
<!-- Reinforcement\nLearning -->
<g id="node10" class="node"><title>Reinforcement\nLearning</title>
<g id="a_node10"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ReinforcementLearning" xlink:title="Reinforcement learning (RL) is an area of machine learning concerned with how
software agents ought to take actions in an environment in order to maximize
the notion of cumulative reward [from Wikipedia]

">
<text text-anchor="middle" x="71" y="-398.6" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="18.00">Reinforcement</text>
<text text-anchor="middle" x="71" y="-378.6" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="18.00">Learning</text>
</a>
</g>
</g>
<!-- Model Free -->
<g id="node11" class="node"><title>Model Free</title>
<g id="a_node11"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ModelFree" xlink:title="In model free reinforcement learning, the agent directly tries to predict the
value/policy without having or trying to model the environment

">
<path fill="#ffe6cc" stroke="black" stroke-width="2" d="M252.5,-1022C252.5,-1022 196.5,-1022 196.5,-1022 190.5,-1022 184.5,-1016 184.5,-1010 184.5,-1010 184.5,-998 184.5,-998 184.5,-992 190.5,-986 196.5,-986 196.5,-986 252.5,-986 252.5,-986 258.5,-986 264.5,-992 264.5,-998 264.5,-998 264.5,-1010 264.5,-1010 264.5,-1016 258.5,-1022 252.5,-1022"/>
<text text-anchor="middle" x="224.5" y="-1000.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Model Free</text>
</a>
</g>
</g>
<!-- Reinforcement\nLearning&#45;&gt;Model Free -->
<g id="edge86" class="edge"><title>Reinforcement\nLearning&#45;&gt;Model Free</title>
<path fill="none" stroke="black" d="M78.0325,-417.329C102.057,-514.222 190.867,-872.391 216.519,-975.847"/>
<polygon fill="black" stroke="black" points="213.196,-976.988 219,-985.851 219.99,-975.303 213.196,-976.988"/>
</g>
<!-- Model Based -->
<g id="node51" class="node"><title>Model Based</title>
<g id="a_node51"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ModelBased" xlink:title="In model&#45;based reinforcement learning, the agent uses the experience to try to
model the environment, and then uses the model to predict the value/policy

">
<path fill="#ffe6cc" stroke="black" stroke-width="2" d="M258,-411C258,-411 191,-411 191,-411 185,-411 179,-405 179,-399 179,-399 179,-387 179,-387 179,-381 185,-375 191,-375 191,-375 258,-375 258,-375 264,-375 270,-381 270,-387 270,-387 270,-399 270,-399 270,-405 264,-411 258,-411"/>
<text text-anchor="middle" x="224.5" y="-389.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Model Based</text>
</a>
</g>
</g>
<!-- Reinforcement\nLearning&#45;&gt;Model Based -->
<g id="edge87" class="edge"><title>Reinforcement\nLearning&#45;&gt;Model Based</title>
<path fill="none" stroke="black" d="M142.428,-393C151.217,-393 160.103,-393 168.608,-393"/>
<polygon fill="black" stroke="black" points="168.762,-396.5 178.762,-393 168.762,-389.5 168.762,-396.5"/>
</g>
<!-- Meta&#45;RL -->
<g id="node69" class="node"><title>Meta&#45;RL</title>
<g id="a_node69"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MetaRL" xlink:title="In meta reinforcement learning, the agent is trained over distribution of
tasks, and with the knowledge it tries to solve new unseen but related task.

(2001)">
<path fill="#ffe6cc" stroke="black" stroke-width="2" d="M672,-179C672,-179 632,-179 632,-179 626,-179 620,-173 620,-167 620,-167 620,-155 620,-155 620,-149 626,-143 632,-143 632,-143 672,-143 672,-143 678,-143 684,-149 684,-155 684,-155 684,-167 684,-167 684,-173 678,-179 672,-179"/>
<text text-anchor="middle" x="652" y="-157.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Meta&#45;RL</text>
</a>
</g>
</g>
<!-- Reinforcement\nLearning&#45;&gt;Meta&#45;RL -->
<g id="edge88" class="edge"><title>Reinforcement\nLearning&#45;&gt;Meta&#45;RL</title>
<path fill="none" stroke="black" d="M77.9206,-368.755C92.5797,-315.295 135.885,-192 223.5,-192 223.5,-192 223.5,-192 516.5,-192 548.585,-192 583.901,-183.52 610.291,-175.43"/>
<polygon fill="black" stroke="black" points="611.469,-178.728 619.942,-172.367 609.352,-172.056 611.469,-178.728"/>
</g>
<!-- Value Gradient -->
<g id="node12" class="node"><title>Value Gradient</title>
<g id="a_node12"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ValueGradient" xlink:title="The algorithm is learning the value function of each state or state&#45;action.
The policy is implicit, usually by just selecting the best value

">
<path fill="#ffe6cc" stroke="black" stroke-width="2" d="M397.5,-1384C397.5,-1384 320.5,-1384 320.5,-1384 314.5,-1384 308.5,-1378 308.5,-1372 308.5,-1372 308.5,-1360 308.5,-1360 308.5,-1354 314.5,-1348 320.5,-1348 320.5,-1348 397.5,-1348 397.5,-1348 403.5,-1348 409.5,-1354 409.5,-1360 409.5,-1360 409.5,-1372 409.5,-1372 409.5,-1378 403.5,-1384 397.5,-1384"/>
<text text-anchor="middle" x="359" y="-1362.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Value Gradient</text>
</a>
</g>
</g>
<!-- Model Free&#45;&gt;Value Gradient -->
<g id="edge9" class="edge"><title>Model Free&#45;&gt;Value Gradient</title>
<path fill="none" stroke="black" d="M227.091,-1022.26C231.864,-1070.07 248.969,-1201.86 299,-1298 307.108,-1313.58 319.468,-1328.59 330.784,-1340.52"/>
<polygon fill="black" stroke="black" points="328.513,-1343.2 338.014,-1347.89 333.511,-1338.3 328.513,-1343.2"/>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic -->
<g id="node13" class="node"><title>Policy Gradient\n/Actor&#45;Critic</title>
<g id="a_node13"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PolicyGradientActorCritic" xlink:title="The algorithm works directly to optimize the policy, with or without value
function. If the value function is learned in addition to the policy, we would
get Actor&#45;Critic algorithm. Most policy gradient algorithms are Actor&#45;Critic.
The Critic updates value function parameters w and depending on the algorithm
it could be action&#45;value Q(a|s;w) or state&#45;value V(s;w). The Actor updates
policy parameters θ, in the direction suggested by the critic, π(a|s;θ). [from
Lilian Weng&#39; blog]

">
<path fill="#ffe6cc" stroke="black" stroke-width="2" d="M399,-1022C399,-1022 319,-1022 319,-1022 313,-1022 307,-1016 307,-1010 307,-1010 307,-998 307,-998 307,-992 313,-986 319,-986 319,-986 399,-986 399,-986 405,-986 411,-992 411,-998 411,-998 411,-1010 411,-1010 411,-1016 405,-1022 399,-1022"/>
<text text-anchor="middle" x="359" y="-1007.4" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Policy Gradient</text>
<text text-anchor="middle" x="359" y="-994.4" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">/Actor&#45;Critic</text>
</a>
</g>
</g>
<!-- Model Free&#45;&gt;Policy Gradient\n/Actor&#45;Critic -->
<g id="edge10" class="edge"><title>Model Free&#45;&gt;Policy Gradient\n/Actor&#45;Critic</title>
<path fill="none" stroke="black" d="M264.538,-1004C274.707,-1004 285.912,-1004 296.925,-1004"/>
<polygon fill="black" stroke="black" points="296.95,-1007.5 306.95,-1004 296.95,-1000.5 296.95,-1007.5"/>
</g>
<!-- SARSA -->
<g id="node25" class="node"><title>SARSA</title>
<g id="a_node25"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#SARSA" xlink:title="SARSA (State&#45;Action&#45;Reward&#45;State&#45;Action) is an on&#45;policy TD control method

(1994)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M533,-1465C533,-1465 498,-1465 498,-1465 492,-1465 486,-1459 486,-1453 486,-1453 486,-1441 486,-1441 486,-1435 492,-1429 498,-1429 498,-1429 533,-1429 533,-1429 539,-1429 545,-1435 545,-1441 545,-1441 545,-1453 545,-1453 545,-1459 539,-1465 533,-1465"/>
<text text-anchor="middle" x="515.5" y="-1443.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">SARSA</text>
</a>
</g>
</g>
<!-- Value Gradient&#45;&gt;SARSA -->
<g id="edge17" class="edge"><title>Value Gradient&#45;&gt;SARSA</title>
<path fill="none" stroke="black" d="M387.075,-1384.22C404.147,-1395.3 426.884,-1409.33 448,-1420 456.996,-1424.55 466.948,-1428.92 476.352,-1432.77"/>
<polygon fill="black" stroke="black" points="475.306,-1436.12 485.891,-1436.58 477.903,-1429.62 475.306,-1436.12"/>
</g>
<!-- Q&#45;learning -->
<g id="node26" class="node"><title>Q&#45;learning</title>
<g id="a_node26"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#Qlearning" xlink:title="Q&#45;learning an off&#45;policy TD control method. Unlike SARSA, it doesn&#39;t follow
the policy to find the next action but rather chooses most optimal action in a
greedy fashion

(1989)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M541.5,-1357C541.5,-1357 489.5,-1357 489.5,-1357 483.5,-1357 477.5,-1351 477.5,-1345 477.5,-1345 477.5,-1333 477.5,-1333 477.5,-1327 483.5,-1321 489.5,-1321 489.5,-1321 541.5,-1321 541.5,-1321 547.5,-1321 553.5,-1327 553.5,-1333 553.5,-1333 553.5,-1345 553.5,-1345 553.5,-1351 547.5,-1357 541.5,-1357"/>
<text text-anchor="middle" x="515.5" y="-1335.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Q&#45;learning</text>
</a>
</g>
</g>
<!-- Value Gradient&#45;&gt;Q&#45;learning -->
<g id="edge18" class="edge"><title>Value Gradient&#45;&gt;Q&#45;learning</title>
<path fill="none" stroke="black" d="M409.735,-1357.31C428.108,-1354.1 448.87,-1350.47 467.037,-1347.29"/>
<polygon fill="black" stroke="black" points="467.864,-1350.7 477.112,-1345.53 466.659,-1343.81 467.864,-1350.7"/>
</g>
<!-- TD&#45;Gammon -->
<g id="node27" class="node"><title>TD&#45;Gammon</title>
<g id="a_node27"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#TDGammon" xlink:title="TD&#45;Gammon is a model&#45;free reinforcement learning algorithm similar to
Q&#45;learning, and uses a multi&#45;layer perceptron with one hidden layer as the
value function approximator. It learns the game entirely by playing against
itself and achieves superhuman level of play.

(1995)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M548,-1411C548,-1411 483,-1411 483,-1411 477,-1411 471,-1405 471,-1399 471,-1399 471,-1387 471,-1387 471,-1381 477,-1375 483,-1375 483,-1375 548,-1375 548,-1375 554,-1375 560,-1381 560,-1387 560,-1387 560,-1399 560,-1399 560,-1405 554,-1411 548,-1411"/>
<text text-anchor="middle" x="515.5" y="-1389.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">TD&#45;Gammon</text>
</a>
</g>
</g>
<!-- Value Gradient&#45;&gt;TD&#45;Gammon -->
<g id="edge19" class="edge"><title>Value Gradient&#45;&gt;TD&#45;Gammon</title>
<path fill="none" stroke="black" d="M409.735,-1374.69C426.026,-1377.54 444.194,-1380.71 460.766,-1383.61"/>
<polygon fill="black" stroke="black" points="460.394,-1387.1 470.848,-1385.37 461.599,-1380.2 460.394,-1387.1"/>
</g>
<!-- A3C -->
<g id="node23" class="node"><title>A3C</title>
<g id="a_node23"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#A3C" xlink:title="Asynchronous Advantage Actor&#45;Critic (A3C) is a classic policy gradient method
with the special focus on parallel training. In A3C, the critics learn the
state&#45;value function, V(s;w), while multiple actors are trained in parallel
and get synced with global parameters from time to time. Hence, A3C is good
for parallel training by default, i.e. on one machine with multi&#45;core CPU.
[from Lilian Weng&#39; blog]

(2016)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M951.5,-1126C951.5,-1126 921.5,-1126 921.5,-1126 915.5,-1126 909.5,-1120 909.5,-1114 909.5,-1114 909.5,-1102 909.5,-1102 909.5,-1096 915.5,-1090 921.5,-1090 921.5,-1090 951.5,-1090 951.5,-1090 957.5,-1090 963.5,-1096 963.5,-1102 963.5,-1102 963.5,-1114 963.5,-1114 963.5,-1120 957.5,-1126 951.5,-1126"/>
<text text-anchor="middle" x="936.5" y="-1104.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">A3C</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;A3C -->
<g id="edge41" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;A3C</title>
<path fill="none" stroke="black" d="M378.948,-1022.01C395.732,-1036.86 421.606,-1057.18 448,-1068 475.655,-1079.34 484.612,-1078 514.5,-1078 514.5,-1078 514.5,-1078 817.5,-1078 845.672,-1078 876.493,-1086.13 899.569,-1093.93"/>
<polygon fill="black" stroke="black" points="898.677,-1097.32 909.271,-1097.34 900.999,-1090.72 898.677,-1097.32"/>
</g>
<!-- REINFORCE -->
<g id="node37" class="node"><title>REINFORCE</title>
<g id="a_node37"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#REINFORCE" xlink:title="REINFORCE (Monte&#45;Carlo policy gradient) is a pure policy gradient algorithm
that works without a value function. The agent collects a trajectory of one
episode using its current policy, and uses the returns to update the policy
parameter

(1992)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M548,-1059C548,-1059 483,-1059 483,-1059 477,-1059 471,-1053 471,-1047 471,-1047 471,-1035 471,-1035 471,-1029 477,-1023 483,-1023 483,-1023 548,-1023 548,-1023 554,-1023 560,-1029 560,-1035 560,-1035 560,-1047 560,-1047 560,-1053 554,-1059 548,-1059"/>
<text text-anchor="middle" x="515.5" y="-1037.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">REINFORCE</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;REINFORCE -->
<g id="edge37" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;REINFORCE</title>
<path fill="none" stroke="black" d="M411.017,-1016.22C427.031,-1020.05 444.751,-1024.3 460.935,-1028.17"/>
<polygon fill="black" stroke="black" points="460.245,-1031.61 470.785,-1030.53 461.875,-1024.8 460.245,-1031.61"/>
</g>
<!-- DPG -->
<g id="node38" class="node"><title>DPG</title>
<g id="a_node38"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DPG" xlink:title="Deterministic Policy Gradient. Abstract: In this paper we consider
deterministic policy gradient algorithms for reinforcement learning with
continuous actions. The deterministic policy gradient has a particularly
appealing form: it is the expected gradient of the action&#45;value function. This
simple form means that the deterministic policy gradient can be estimated much
more efficiently than the usual stochastic policy gradient. To ensure adequate
exploration, we introduce an off&#45;policy actor&#45;critic algorithm that learns a
deterministic target policy from an exploratory behaviour policy. We
demonstrate that deterministic policy gradient algorithms can significantly
outperform their stochastic counterparts in high&#45;dimensional action spaces.

(2014)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1039C831.5,-1039 801.5,-1039 801.5,-1039 795.5,-1039 789.5,-1033 789.5,-1027 789.5,-1027 789.5,-1015 789.5,-1015 789.5,-1009 795.5,-1003 801.5,-1003 801.5,-1003 831.5,-1003 831.5,-1003 837.5,-1003 843.5,-1009 843.5,-1015 843.5,-1015 843.5,-1027 843.5,-1027 843.5,-1033 837.5,-1039 831.5,-1039"/>
<text text-anchor="middle" x="816.5" y="-1017.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DPG</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;DPG -->
<g id="edge38" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;DPG</title>
<path fill="none" stroke="black" d="M411.366,-1005.92C504.442,-1009.39 698.784,-1016.64 779.329,-1019.65"/>
<polygon fill="black" stroke="black" points="779.347,-1023.15 789.47,-1020.03 779.608,-1016.16 779.347,-1023.15"/>
</g>
<!-- TRPO -->
<g id="node39" class="node"><title>TRPO</title>
<g id="a_node39"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#TRPO" xlink:title="Trust Region Policy Optimization (TRPO) improves training stability by
enforcing a KL divergence constraint to avoid parameter updates that change
the policy too much at one step.

(2015)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1133C831.5,-1133 801.5,-1133 801.5,-1133 795.5,-1133 789.5,-1127 789.5,-1121 789.5,-1121 789.5,-1109 789.5,-1109 789.5,-1103 795.5,-1097 801.5,-1097 801.5,-1097 831.5,-1097 831.5,-1097 837.5,-1097 843.5,-1103 843.5,-1109 843.5,-1109 843.5,-1121 843.5,-1121 843.5,-1127 837.5,-1133 831.5,-1133"/>
<text text-anchor="middle" x="816.5" y="-1111.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">TRPO</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;TRPO -->
<g id="edge39" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;TRPO</title>
<path fill="none" stroke="black" d="M370.001,-1022.06C381.523,-1040.9 402.343,-1069.54 429,-1083 545.615,-1141.9 707.498,-1129.9 779.03,-1120.64"/>
<polygon fill="black" stroke="black" points="779.874,-1124.05 789.313,-1119.24 778.933,-1117.12 779.874,-1124.05"/>
</g>
<!-- GAE -->
<g id="node40" class="node"><title>GAE</title>
<g id="a_node40"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#GAE" xlink:title="Generalized Advantage Estimation

(2015)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1205C831.5,-1205 801.5,-1205 801.5,-1205 795.5,-1205 789.5,-1199 789.5,-1193 789.5,-1193 789.5,-1181 789.5,-1181 789.5,-1175 795.5,-1169 801.5,-1169 801.5,-1169 831.5,-1169 831.5,-1169 837.5,-1169 843.5,-1175 843.5,-1181 843.5,-1181 843.5,-1193 843.5,-1193 843.5,-1199 837.5,-1205 831.5,-1205"/>
<text text-anchor="middle" x="816.5" y="-1183.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">GAE</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;GAE -->
<g id="edge40" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;GAE</title>
<path fill="none" stroke="black" d="M369.169,-1022.12C382.786,-1047.04 410.798,-1091.39 448,-1114 555.335,-1179.23 709.823,-1187.59 779.272,-1187.76"/>
<polygon fill="black" stroke="black" points="779.288,-1191.26 789.277,-1187.73 779.266,-1184.26 779.288,-1191.26"/>
</g>
<!-- ACKTR -->
<g id="node41" class="node"><title>ACKTR</title>
<g id="a_node41"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ACKTR" xlink:title="Actor Critic using Kronecker&#45;Factored Trust Region (ACKTR) is applying trust
region optimization to deep reinforcement learning using a recently proposed
Kronecker&#45;factored approximation to the curvature.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1165,-952C1165,-952 1130,-952 1130,-952 1124,-952 1118,-946 1118,-940 1118,-940 1118,-928 1118,-928 1118,-922 1124,-916 1130,-916 1130,-916 1165,-916 1165,-916 1171,-916 1177,-922 1177,-928 1177,-928 1177,-940 1177,-940 1177,-946 1171,-952 1165,-952"/>
<text text-anchor="middle" x="1147.5" y="-930.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">ACKTR</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;ACKTR -->
<g id="edge42" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;ACKTR</title>
<path fill="none" stroke="black" d="M379.784,-985.847C407.645,-962.243 461.036,-924 514.5,-924 514.5,-924 514.5,-924 937.5,-924 996.942,-924 1065.78,-928.086 1107.72,-931.034"/>
<polygon fill="black" stroke="black" points="1107.56,-934.531 1117.78,-931.756 1108.06,-927.549 1107.56,-934.531"/>
</g>
<!-- SVPG -->
<g id="node42" class="node"><title>SVPG</title>
<g id="a_node42"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#SVPG" xlink:title="Stein Variational Policy Gradient (SVPG)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-1006C1162.5,-1006 1132.5,-1006 1132.5,-1006 1126.5,-1006 1120.5,-1000 1120.5,-994 1120.5,-994 1120.5,-982 1120.5,-982 1120.5,-976 1126.5,-970 1132.5,-970 1132.5,-970 1162.5,-970 1162.5,-970 1168.5,-970 1174.5,-976 1174.5,-982 1174.5,-982 1174.5,-994 1174.5,-994 1174.5,-1000 1168.5,-1006 1162.5,-1006"/>
<text text-anchor="middle" x="1147.5" y="-984.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">SVPG</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;SVPG -->
<g id="edge43" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;SVPG</title>
<path fill="none" stroke="black" d="M404.843,-985.989C435.266,-975.372 476.68,-964 514.5,-964 514.5,-964 514.5,-964 937.5,-964 1003.98,-964 1021.49,-957.67 1087,-969 1094.73,-970.336 1102.84,-972.434 1110.49,-974.756"/>
<polygon fill="black" stroke="black" points="1109.64,-978.158 1120.23,-977.892 1111.79,-971.495 1109.64,-978.158"/>
</g>
<!-- Reactor -->
<g id="node43" class="node"><title>Reactor</title>
<g id="a_node43"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#Reactor" xlink:title="From the abstract: In this work we present a new agent architecture, called
Reactor, which combines multiple algorithmic and architectural contributions
to produce an agent with higher sample&#45;efficiency than Prioritized Dueling DQN
(Wang et al., 2016) and Categorical DQN (Bellemare et al., 2017), while giving
better run&#45;time performance than A3C (Mnih et al., 2016). Our first
contribution is a new policy evaluation algorithm called Distributional
Retrace, which brings multi&#45;step off&#45;policy updates to the distributional
reinforcement learning setting. The same approach can be used to convert
several classes of multi&#45;step policy evaluation algorithms designed for
expected value evaluation into distributional ones. Next, we introduce the
β&#45;leave&#45;one&#45;out policy gradient algorithm which improves the trade&#45;off between
variance and bias by using action values as a baseline. Our final algorithmic
contribution is a new prioritized replay algorithm for sequences, which
exploits the temporal locality of neighboring observations for more efficient
replay prioritization. Using the Atari 2600 benchmarks, we show that each of
these innovations contribute to both the sample efficiency and final agent
performance. Finally, we demonstrate that Reactor reaches state&#45;of&#45;the&#45;art
performance after 200 million frames and less than a day of training.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1166.5,-1060C1166.5,-1060 1128.5,-1060 1128.5,-1060 1122.5,-1060 1116.5,-1054 1116.5,-1048 1116.5,-1048 1116.5,-1036 1116.5,-1036 1116.5,-1030 1122.5,-1024 1128.5,-1024 1128.5,-1024 1166.5,-1024 1166.5,-1024 1172.5,-1024 1178.5,-1030 1178.5,-1036 1178.5,-1036 1178.5,-1048 1178.5,-1048 1178.5,-1054 1172.5,-1060 1166.5,-1060"/>
<text text-anchor="middle" x="1147.5" y="-1038.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Reactor</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;Reactor -->
<g id="edge44" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;Reactor</title>
<path fill="none" stroke="black" d="M411.285,-994C441.255,-989 479.88,-984 514.5,-984 514.5,-984 514.5,-984 733.5,-984 871.632,-984 1033.25,-1016.21 1106.63,-1032.62"/>
<polygon fill="black" stroke="black" points="1105.87,-1036.04 1116.39,-1034.83 1107.41,-1029.21 1105.87,-1036.04"/>
</g>
<!-- SAC -->
<g id="node44" class="node"><title>SAC</title>
<g id="a_node44"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#SAC" xlink:title="Soft Actor Critic (SAC) is an algorithm that optimizes a stochastic policy in
an off&#45;policy way, forming a bridge between stochastic policy optimization and
DDPG&#45;style approaches.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-929C1317.5,-929 1287.5,-929 1287.5,-929 1281.5,-929 1275.5,-923 1275.5,-917 1275.5,-917 1275.5,-905 1275.5,-905 1275.5,-899 1281.5,-893 1287.5,-893 1287.5,-893 1317.5,-893 1317.5,-893 1323.5,-893 1329.5,-899 1329.5,-905 1329.5,-905 1329.5,-917 1329.5,-917 1329.5,-923 1323.5,-929 1317.5,-929"/>
<text text-anchor="middle" x="1302.5" y="-907.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">SAC</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;SAC -->
<g id="edge45" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;SAC</title>
<path fill="none" stroke="black" d="M364.068,-985.688C371.547,-956.598 390.385,-900.238 429,-872 460.142,-849.227 475.92,-857 514.5,-857 514.5,-857 514.5,-857 1148.5,-857 1190.88,-857 1236.28,-875.648 1266.52,-891.027"/>
<polygon fill="black" stroke="black" points="1264.92,-894.137 1275.4,-895.67 1268.16,-887.934 1264.92,-894.137"/>
</g>
<!-- MPO -->
<g id="node45" class="node"><title>MPO</title>
<g id="a_node45"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MPO" xlink:title="Maximum a Posteriori Policy Optimization (MPO) is an RL method that combines
the sample efficiency of off&#45;policy methods with the scalability and
hyperparameter robustness of on&#45;policy methods. It is an EM style method,
which alternates an E&#45;step that re&#45;weights state&#45;action samples with an M step
that updates a deep neural network with supervised training. MPO achieves
state of the art results on many continuous control tasks while using an order
of magnitude fewer samples when compared with PPO

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-875C1317.5,-875 1287.5,-875 1287.5,-875 1281.5,-875 1275.5,-869 1275.5,-863 1275.5,-863 1275.5,-851 1275.5,-851 1275.5,-845 1281.5,-839 1287.5,-839 1287.5,-839 1317.5,-839 1317.5,-839 1323.5,-839 1329.5,-845 1329.5,-851 1329.5,-851 1329.5,-863 1329.5,-863 1329.5,-869 1323.5,-875 1317.5,-875"/>
<text text-anchor="middle" x="1302.5" y="-853.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MPO</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;MPO -->
<g id="edge46" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;MPO</title>
<path fill="none" stroke="black" d="M370.603,-985.804C392.59,-950.687 446.792,-877 514.5,-877 514.5,-877 514.5,-877 1148.5,-877 1188.81,-877 1234.6,-870.089 1265.5,-864.392"/>
<polygon fill="black" stroke="black" points="1266.2,-867.821 1275.37,-862.518 1264.9,-860.943 1266.2,-867.821"/>
</g>
<!-- IMPALA -->
<g id="node46" class="node"><title>IMPALA</title>
<g id="a_node46"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#IMPALA" xlink:title="Importance Weighted Actor&#45;Learner Architecture (IMPALA)

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1321.5,-1094C1321.5,-1094 1283.5,-1094 1283.5,-1094 1277.5,-1094 1271.5,-1088 1271.5,-1082 1271.5,-1082 1271.5,-1070 1271.5,-1070 1271.5,-1064 1277.5,-1058 1283.5,-1058 1283.5,-1058 1321.5,-1058 1321.5,-1058 1327.5,-1058 1333.5,-1064 1333.5,-1070 1333.5,-1070 1333.5,-1082 1333.5,-1082 1333.5,-1088 1327.5,-1094 1321.5,-1094"/>
<text text-anchor="middle" x="1302.5" y="-1072.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">IMPALA</text>
</a>
</g>
</g>
<!-- Policy Gradient\n/Actor&#45;Critic&#45;&gt;IMPALA -->
<g id="edge47" class="edge"><title>Policy Gradient\n/Actor&#45;Critic&#45;&gt;IMPALA</title>
<path fill="none" stroke="black" d="M363.515,-1022C374.921,-1074.55 416.022,-1224 514.5,-1224 514.5,-1224 514.5,-1224 871,-1224 921.271,-1224 934,-1223.78 984,-1229 1038.21,-1234.66 1050.68,-1244.54 1105,-1249 1142.65,-1252.09 1156.83,-1267.08 1190,-1249 1204.9,-1240.88 1259.05,-1149.6 1286.02,-1103.01"/>
<polygon fill="black" stroke="black" points="1289.22,-1104.47 1291.19,-1094.06 1283.16,-1100.97 1289.22,-1104.47"/>
</g>
<!-- DQN -->
<g id="node14" class="node"><title>DQN</title>
<g id="a_node14"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DQN" xlink:title="Deep Q Network (DQN) is Q&#45;Learning with deep neural network as state&#45;action
value estimator and uses a replay buffer to sample experiences from previous
trajectories to make learning more stable.

(2013)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1376C831.5,-1376 801.5,-1376 801.5,-1376 795.5,-1376 789.5,-1370 789.5,-1364 789.5,-1364 789.5,-1352 789.5,-1352 789.5,-1346 795.5,-1340 801.5,-1340 801.5,-1340 831.5,-1340 831.5,-1340 837.5,-1340 843.5,-1346 843.5,-1352 843.5,-1352 843.5,-1364 843.5,-1364 843.5,-1370 837.5,-1376 831.5,-1376"/>
<text text-anchor="middle" x="816.5" y="-1354.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DQN</text>
</a>
</g>
</g>
<!-- DDPG -->
<g id="node15" class="node"><title>DDPG</title>
<g id="a_node15"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DDPG" xlink:title="Deep Deterministic Policy Gradient (DDPG).

(2015)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-797C831.5,-797 801.5,-797 801.5,-797 795.5,-797 789.5,-791 789.5,-785 789.5,-785 789.5,-773 789.5,-773 789.5,-767 795.5,-761 801.5,-761 801.5,-761 831.5,-761 831.5,-761 837.5,-761 843.5,-767 843.5,-773 843.5,-773 843.5,-785 843.5,-785 843.5,-791 837.5,-797 831.5,-797"/>
<text text-anchor="middle" x="816.5" y="-775.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DDPG</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;DDPG -->
<g id="edge11" class="edge"><title>DQN&#45;&gt;DDPG</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M803.515,-1339.94C795.852,-1328.45 786.588,-1312.97 781,-1298 751.993,-1220.3 692.058,-1009.9 716,-930.5 730.845,-881.267 767.498,-833.34 792.19,-804.996"/>
<polygon fill="darkgray" stroke="darkgray" points="795.097,-806.993 799.12,-797.192 789.863,-802.345 795.097,-806.993"/>
<text text-anchor="middle" x="732.5" y="-933.5" font-family="sans-serif" font-size="10.00" fill="darkgray">replay buffer</text>
</g>
<!-- ACER -->
<g id="node16" class="node"><title>ACER</title>
<g id="a_node16"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ACER" xlink:title="Actor&#45;Critic with Experience Replay (ACER) combines several ideas of previous
algorithms: it uses multiple workers (as A2C), implements a replay buffer (as
in DQN), uses Retrace for Q&#45;value estimation, importance sampling and a trust
region. ACER is A3C&#39;s off&#45;policy counterpart. ACER proposes several designs to
overcome the major obstacle to making A3C off policy, that is how to control
the stability of the off&#45;policy estimator. (source: Lilian Weng&#39;s blog)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-1186C1162.5,-1186 1132.5,-1186 1132.5,-1186 1126.5,-1186 1120.5,-1180 1120.5,-1174 1120.5,-1174 1120.5,-1162 1120.5,-1162 1120.5,-1156 1126.5,-1150 1132.5,-1150 1132.5,-1150 1162.5,-1150 1162.5,-1150 1168.5,-1150 1174.5,-1156 1174.5,-1162 1174.5,-1162 1174.5,-1174 1174.5,-1174 1174.5,-1180 1168.5,-1186 1162.5,-1186"/>
<text text-anchor="middle" x="1147.5" y="-1164.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">ACER</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;ACER -->
<g id="edge12" class="edge"><title>DQN&#45;&gt;ACER</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M827.8,-1339.79C837.29,-1324.16 852.521,-1301.73 870,-1286 941.401,-1221.74 1053.05,-1188.89 1110.17,-1175.5"/>
<polygon fill="darkgray" stroke="darkgray" points="1111.2,-1178.85 1120.18,-1173.22 1109.65,-1172.03 1111.2,-1178.85"/>
<text text-anchor="middle" x="936.5" y="-1272" font-family="sans-serif" font-size="10.00" fill="darkgray">replay buffer</text>
</g>
<!-- DDQN -->
<g id="node17" class="node"><title>DDQN</title>
<g id="a_node17"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DDQN" xlink:title="Double DQN adds another neural network, making separate network for policy and
target. The target network is only updated after certain number of
steps/episodes. This makes the learning more stable.

(2015)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1502C831.5,-1502 801.5,-1502 801.5,-1502 795.5,-1502 789.5,-1496 789.5,-1490 789.5,-1490 789.5,-1478 789.5,-1478 789.5,-1472 795.5,-1466 801.5,-1466 801.5,-1466 831.5,-1466 831.5,-1466 837.5,-1466 843.5,-1472 843.5,-1478 843.5,-1478 843.5,-1490 843.5,-1490 843.5,-1496 837.5,-1502 831.5,-1502"/>
<text text-anchor="middle" x="816.5" y="-1480.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DDQN</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;DDQN -->
<g id="edge22" class="edge"><title>DQN&#45;&gt;DDQN</title>
<path fill="none" stroke="black" d="M789.203,-1367.38C769.546,-1375.74 744.463,-1390.09 732.5,-1412 724.833,-1426.04 723.692,-1434.64 732.5,-1448 742.941,-1463.83 761.946,-1472.73 779.242,-1477.72"/>
<polygon fill="black" stroke="black" points="778.597,-1481.16 789.15,-1480.22 780.313,-1474.38 778.597,-1481.16"/>
</g>
<!-- DQN+HER -->
<g id="node19" class="node"><title>DQN+HER</title>
<g id="a_node19"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DQNHER" xlink:title="DQN with Hindsight Experience Replay (HER)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1174,-1386C1174,-1386 1121,-1386 1121,-1386 1115,-1386 1109,-1380 1109,-1374 1109,-1374 1109,-1362 1109,-1362 1109,-1356 1115,-1350 1121,-1350 1121,-1350 1174,-1350 1174,-1350 1180,-1350 1186,-1356 1186,-1362 1186,-1362 1186,-1374 1186,-1374 1186,-1380 1180,-1386 1174,-1386"/>
<text text-anchor="middle" x="1147.5" y="-1364.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DQN+HER</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;DQN+HER -->
<g id="edge26" class="edge"><title>DQN&#45;&gt;DQN+HER</title>
<path fill="none" stroke="black" d="M843.591,-1358.79C898.742,-1360.47 1027.87,-1364.39 1098.41,-1366.54"/>
<polygon fill="black" stroke="black" points="1098.69,-1370.05 1108.8,-1366.85 1098.91,-1363.05 1098.69,-1370.05"/>
</g>
<!-- APE&#45;X DQN -->
<g id="node21" class="node"><title>APE&#45;X DQN</title>
<g id="a_node21"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#APEXDQN" xlink:title="DQN with Distributed Prioritized Experience Replay

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1332,-1423C1332,-1423 1273,-1423 1273,-1423 1267,-1423 1261,-1417 1261,-1411 1261,-1411 1261,-1399 1261,-1399 1261,-1393 1267,-1387 1273,-1387 1273,-1387 1332,-1387 1332,-1387 1338,-1387 1344,-1393 1344,-1399 1344,-1399 1344,-1411 1344,-1411 1344,-1417 1338,-1423 1332,-1423"/>
<text text-anchor="middle" x="1302.5" y="-1401.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">APE&#45;X DQN</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;APE&#45;X DQN -->
<g id="edge28" class="edge"><title>DQN&#45;&gt;APE&#45;X DQN</title>
<path fill="none" stroke="black" d="M843.666,-1361.52C852.074,-1362.64 861.429,-1363.88 870,-1365 974.407,-1378.62 1000.12,-1385.65 1105,-1395 1154.35,-1399.4 1210.85,-1402.02 1250.57,-1403.47"/>
<polygon fill="black" stroke="black" points="1250.68,-1406.98 1260.79,-1403.83 1250.92,-1399.98 1250.68,-1406.98"/>
</g>
<!-- DRQN -->
<g id="node28" class="node"><title>DRQN</title>
<g id="a_node28"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DRQN" xlink:title="Deep Recurrent Q&#45;Learning. Adding recurrency to a Deep Q&#45;Network (DQN) by
replacing the first post&#45;convolutional fully&#45;connected layer with a recurrent
LSTM

(2015)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1448C831.5,-1448 801.5,-1448 801.5,-1448 795.5,-1448 789.5,-1442 789.5,-1436 789.5,-1436 789.5,-1424 789.5,-1424 789.5,-1418 795.5,-1412 801.5,-1412 801.5,-1412 831.5,-1412 831.5,-1412 837.5,-1412 843.5,-1418 843.5,-1424 843.5,-1424 843.5,-1436 843.5,-1436 843.5,-1442 837.5,-1448 831.5,-1448"/>
<text text-anchor="middle" x="816.5" y="-1426.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DRQN</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;DRQN -->
<g id="edge21" class="edge"><title>DQN&#45;&gt;DRQN</title>
<path fill="none" stroke="black" d="M816.5,-1376.28C816.5,-1384.83 816.5,-1393.37 816.5,-1401.92"/>
<polygon fill="black" stroke="black" points="813,-1401.95 816.5,-1411.95 820,-1401.95 813,-1401.95"/>
</g>
<!-- PER -->
<g id="node29" class="node"><title>PER</title>
<g id="a_node29"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PER" xlink:title="Prioritized Experience Replay (PER) improves data efficiency by replaying
transitions from which there is more to learn more often

(2015)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-1561C831.5,-1561 801.5,-1561 801.5,-1561 795.5,-1561 789.5,-1555 789.5,-1549 789.5,-1549 789.5,-1537 789.5,-1537 789.5,-1531 795.5,-1525 801.5,-1525 801.5,-1525 831.5,-1525 831.5,-1525 837.5,-1525 843.5,-1531 843.5,-1537 843.5,-1537 843.5,-1549 843.5,-1549 843.5,-1555 837.5,-1561 831.5,-1561"/>
<text text-anchor="middle" x="816.5" y="-1539.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">PER</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;PER -->
<g id="edge23" class="edge"><title>DQN&#45;&gt;PER</title>
<path fill="none" stroke="black" d="M789.203,-1367.38C769.546,-1375.74 744.463,-1390.09 732.5,-1412 713.332,-1447.11 711.308,-1468.07 732.5,-1502 742.849,-1518.57 762.038,-1528.62 779.466,-1534.62"/>
<polygon fill="black" stroke="black" points="778.855,-1538.1 789.443,-1537.7 780.922,-1531.41 778.855,-1538.1"/>
</g>
<!-- QR&#45;DQN -->
<g id="node30" class="node"><title>QR&#45;DQN</title>
<g id="a_node30"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#QRDQN" xlink:title="Distributional Reinforcement Learning with Quantile Regression (QR&#45;DQN). In
QR&#45;DQN, distribution of values values are used for each state&#45;action pair
instead of a single mean value

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1168,-1460C1168,-1460 1127,-1460 1127,-1460 1121,-1460 1115,-1454 1115,-1448 1115,-1448 1115,-1436 1115,-1436 1115,-1430 1121,-1424 1127,-1424 1127,-1424 1168,-1424 1168,-1424 1174,-1424 1180,-1430 1180,-1436 1180,-1436 1180,-1448 1180,-1448 1180,-1454 1174,-1460 1168,-1460"/>
<text text-anchor="middle" x="1147.5" y="-1438.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">QR&#45;DQN</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;QR&#45;DQN -->
<g id="edge24" class="edge"><title>DQN&#45;&gt;QR&#45;DQN</title>
<path fill="none" stroke="black" d="M830.509,-1376.07C840.034,-1387.82 854.043,-1402.33 870,-1410 946.52,-1446.77 1048.23,-1448.14 1104.52,-1445.38"/>
<polygon fill="black" stroke="black" points="1105.03,-1448.86 1114.82,-1444.81 1104.64,-1441.87 1105.03,-1448.86"/>
</g>
<!-- C51 -->
<g id="node31" class="node"><title>C51</title>
<g id="a_node31"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#C51" xlink:title="C51 Algorithm. The core idea of Distributional Bellman is to ask the following
questions. If we can model the Distribution of the total future rewards, why
restrict ourselves to the expected value (i.e. Q function)? There are several
benefits to learning an approximate distribution rather than its approximate
expectation. [source: flyyufelix&#39;s blog]

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-1514C1162.5,-1514 1132.5,-1514 1132.5,-1514 1126.5,-1514 1120.5,-1508 1120.5,-1502 1120.5,-1502 1120.5,-1490 1120.5,-1490 1120.5,-1484 1126.5,-1478 1132.5,-1478 1132.5,-1478 1162.5,-1478 1162.5,-1478 1168.5,-1478 1174.5,-1484 1174.5,-1490 1174.5,-1490 1174.5,-1502 1174.5,-1502 1174.5,-1508 1168.5,-1514 1162.5,-1514"/>
<text text-anchor="middle" x="1147.5" y="-1492.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">C51</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;C51 -->
<g id="edge25" class="edge"><title>DQN&#45;&gt;C51</title>
<path fill="none" stroke="black" d="M832.332,-1376.13C849.081,-1396.15 875.725,-1426.56 889,-1434 926.824,-1455.2 1048.66,-1478.81 1110.21,-1489.78"/>
<polygon fill="black" stroke="black" points="1109.8,-1493.26 1120.26,-1491.55 1111.02,-1486.37 1109.8,-1493.26"/>
</g>
<!-- IQN -->
<g id="node32" class="node"><title>IQN</title>
<g id="a_node32"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#IQN" xlink:title="Implicit Quantile Networks (IQN). From the abstract: In this work, we build on
recent advances in distributional reinforcement learning to give a generally
applicable, flexible, and state&#45;of&#45;the&#45;art distributional variant of DQN. We
achieve this by using quantile regression to approximate the full quantile
function for the state&#45;action return distribution. By reparameterizing a
distribution over the sample space, this yields an implicitly defined return
distribution and gives rise to a large class of risk&#45;sensitive policies. We
demonstrate improved performance on the 57 Atari 2600 games in the ALE, and
use our algorithm&#39;s implicitly defined distributions to study the effects of
risk&#45;sensitive policies in Atari games.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-1346C1317.5,-1346 1287.5,-1346 1287.5,-1346 1281.5,-1346 1275.5,-1340 1275.5,-1334 1275.5,-1334 1275.5,-1322 1275.5,-1322 1275.5,-1316 1281.5,-1310 1287.5,-1310 1287.5,-1310 1317.5,-1310 1317.5,-1310 1323.5,-1310 1329.5,-1316 1329.5,-1322 1329.5,-1322 1329.5,-1334 1329.5,-1334 1329.5,-1340 1323.5,-1346 1317.5,-1346"/>
<text text-anchor="middle" x="1302.5" y="-1324.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">IQN</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;IQN -->
<g id="edge27" class="edge"><title>DQN&#45;&gt;IQN</title>
<path fill="none" stroke="black" d="M841.038,-1339.88C849.669,-1334.29 859.826,-1328.86 870,-1326 1006.93,-1287.56 1047.78,-1324.72 1190,-1326 1215.12,-1326.23 1243.43,-1326.74 1265.23,-1327.19"/>
<polygon fill="black" stroke="black" points="1265.3,-1330.69 1275.37,-1327.4 1265.44,-1323.69 1265.3,-1330.69"/>
</g>
<!-- R2D2 -->
<g id="node33" class="node"><title>R2D2</title>
<g id="a_node33"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#R2D2" xlink:title="Recurrent Replay Distributed DQN (R2D2). (from the abstract) Building on the
recent successes of distributed training of RL agents, in this paper we
investigate the training of RNN&#45;based RL agents from distributed prioritized
experience replay. We study the effects of parameter lag resulting in
representational drift and recurrent state staleness and empirically derive an
improved training strategy. Using a single network architecture and fixed set
of hyper&#45;parameters, the resulting agent, Recurrent Replay Distributed DQN,
quadruples the previous state of the art on Atari&#45;57, and matches the state of
the art on DMLab&#45;30. It is the first agent to exceed human&#45;level performance
in 52 of the 57 Atari games.

(2019)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1432.5,-1383C1432.5,-1383 1402.5,-1383 1402.5,-1383 1396.5,-1383 1390.5,-1377 1390.5,-1371 1390.5,-1371 1390.5,-1359 1390.5,-1359 1390.5,-1353 1396.5,-1347 1402.5,-1347 1402.5,-1347 1432.5,-1347 1432.5,-1347 1438.5,-1347 1444.5,-1353 1444.5,-1359 1444.5,-1359 1444.5,-1371 1444.5,-1371 1444.5,-1377 1438.5,-1383 1432.5,-1383"/>
<text text-anchor="middle" x="1417.5" y="-1361.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">R2D2</text>
</a>
</g>
</g>
<!-- DQN&#45;&gt;R2D2 -->
<g id="edge29" class="edge"><title>DQN&#45;&gt;R2D2</title>
<path fill="none" stroke="black" d="M843.776,-1354.08C904.767,-1345.56 1060.61,-1327.22 1190,-1341 1219.39,-1344.13 1225.73,-1350.96 1255,-1355 1297.81,-1360.9 1347.5,-1363.32 1380.33,-1364.31"/>
<polygon fill="black" stroke="black" points="1380.28,-1367.81 1390.37,-1364.59 1380.47,-1360.82 1380.28,-1367.81"/>
</g>
<!-- TD3 -->
<g id="node18" class="node"><title>TD3</title>
<g id="a_node18"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#TD3" xlink:title="Twin Delayed DDPG (TD3). TD3 addresses function approximation error in DDPG by
introducing twin Q&#45;value approximation network and less frequent updates

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-1204C1317.5,-1204 1287.5,-1204 1287.5,-1204 1281.5,-1204 1275.5,-1198 1275.5,-1192 1275.5,-1192 1275.5,-1180 1275.5,-1180 1275.5,-1174 1281.5,-1168 1287.5,-1168 1287.5,-1168 1317.5,-1168 1317.5,-1168 1323.5,-1168 1329.5,-1174 1329.5,-1180 1329.5,-1180 1329.5,-1192 1329.5,-1192 1329.5,-1198 1323.5,-1204 1317.5,-1204"/>
<text text-anchor="middle" x="1302.5" y="-1182.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">TD3</text>
</a>
</g>
</g>
<!-- DDPG&#45;&gt;TD3 -->
<g id="edge53" class="edge"><title>DDPG&#45;&gt;TD3</title>
<path fill="none" stroke="black" d="M843.614,-786.512C924.089,-809.86 1164.42,-881.144 1190,-907 1206.14,-923.311 1245.74,-1082.01 1255,-1103 1263.53,-1122.33 1275.14,-1143.06 1284.63,-1158.97"/>
<polygon fill="black" stroke="black" points="1281.76,-1160.99 1289.93,-1167.73 1287.75,-1157.36 1281.76,-1160.99"/>
</g>
<!-- DDPG+HER -->
<g id="node20" class="node"><title>DDPG+HER</title>
<g id="a_node20"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DDPGHER" xlink:title="Hindsight Experience Replay (HER)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1178,-784C1178,-784 1117,-784 1117,-784 1111,-784 1105,-778 1105,-772 1105,-772 1105,-760 1105,-760 1105,-754 1111,-748 1117,-748 1117,-748 1178,-748 1178,-748 1184,-748 1190,-754 1190,-760 1190,-760 1190,-772 1190,-772 1190,-778 1184,-784 1178,-784"/>
<text text-anchor="middle" x="1147.5" y="-762.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DDPG+HER</text>
</a>
</g>
</g>
<!-- DDPG&#45;&gt;DDPG+HER -->
<g id="edge49" class="edge"><title>DDPG&#45;&gt;DDPG+HER</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M843.591,-777.969C897.768,-775.828 1023.34,-770.867 1094.62,-768.05"/>
<polygon fill="darkgray" stroke="darkgray" points="1094.88,-771.543 1104.73,-767.65 1094.6,-764.548 1094.88,-771.543"/>
</g>
<!-- APE&#45;X DDPG -->
<g id="node22" class="node"><title>APE&#45;X DDPG</title>
<g id="a_node22"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#APEXDDPG" xlink:title="DDPG with Distributed Prioritized Experience Replay

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1336.5,-744C1336.5,-744 1268.5,-744 1268.5,-744 1262.5,-744 1256.5,-738 1256.5,-732 1256.5,-732 1256.5,-720 1256.5,-720 1256.5,-714 1262.5,-708 1268.5,-708 1268.5,-708 1336.5,-708 1336.5,-708 1342.5,-708 1348.5,-714 1348.5,-720 1348.5,-720 1348.5,-732 1348.5,-732 1348.5,-738 1342.5,-744 1336.5,-744"/>
<text text-anchor="middle" x="1302.5" y="-722.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">APE&#45;X DDPG</text>
</a>
</g>
</g>
<!-- DDPG&#45;&gt;APE&#45;X DDPG -->
<g id="edge52" class="edge"><title>DDPG&#45;&gt;APE&#45;X DDPG</title>
<path fill="none" stroke="black" d="M842.727,-760.939C855.884,-752.478 872.619,-743.147 889,-738 1011.9,-699.381 1165.66,-709.093 1246.29,-718.294"/>
<polygon fill="black" stroke="black" points="1245.89,-721.77 1256.23,-719.468 1246.71,-714.819 1245.89,-721.77"/>
</g>
<!-- MADDPG -->
<g id="node47" class="node"><title>MADDPG</title>
<g id="a_node47"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MADDPG" xlink:title="Multi&#45;agent DDPG (MADDPG) extends DDPG to an environment where multiple agents
are coordinating to complete tasks with only local information. In the
viewpoint of one agent, the environment is non&#45;stationary as policies of other
agents are quickly upgraded and remain unknown. MADDPG is an actor&#45;critic
model redesigned particularly for handling such a changing environment and
interactions between agents (from Lilian Weng&#39;s blog)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1170.5,-838C1170.5,-838 1124.5,-838 1124.5,-838 1118.5,-838 1112.5,-832 1112.5,-826 1112.5,-826 1112.5,-814 1112.5,-814 1112.5,-808 1118.5,-802 1124.5,-802 1124.5,-802 1170.5,-802 1170.5,-802 1176.5,-802 1182.5,-808 1182.5,-814 1182.5,-814 1182.5,-826 1182.5,-826 1182.5,-832 1176.5,-838 1170.5,-838"/>
<text text-anchor="middle" x="1147.5" y="-816.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MADDPG</text>
</a>
</g>
</g>
<!-- DDPG&#45;&gt;MADDPG -->
<g id="edge50" class="edge"><title>DDPG&#45;&gt;MADDPG</title>
<path fill="none" stroke="black" d="M843.591,-782.251C899.754,-789.25 1032.63,-805.81 1102.24,-814.484"/>
<polygon fill="black" stroke="black" points="1102.08,-817.991 1112.44,-815.755 1102.95,-811.045 1102.08,-817.991"/>
</g>
<!-- D4PG -->
<g id="node48" class="node"><title>D4PG</title>
<g id="a_node48"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#D4PG" xlink:title="Distributed Distributional Deep Deterministic Policy Gradient (D4PG) adopts
the very successful distributional perspective on reinforcement learning and
adapts it to the continuous control setting. It combines this within a
distributed framework. It also combines this technique with a number of
additional, simple improvements such as the use of N&#45;step returns and
prioritized experience replay [from the paper&#39;s abstract]

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-798C1317.5,-798 1287.5,-798 1287.5,-798 1281.5,-798 1275.5,-792 1275.5,-786 1275.5,-786 1275.5,-774 1275.5,-774 1275.5,-768 1281.5,-762 1287.5,-762 1287.5,-762 1317.5,-762 1317.5,-762 1323.5,-762 1329.5,-768 1329.5,-774 1329.5,-774 1329.5,-786 1329.5,-786 1329.5,-792 1323.5,-798 1317.5,-798"/>
<text text-anchor="middle" x="1302.5" y="-776.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">D4PG</text>
</a>
</g>
</g>
<!-- DDPG&#45;&gt;D4PG -->
<g id="edge51" class="edge"><title>DDPG&#45;&gt;D4PG</title>
<path fill="none" stroke="black" d="M843.87,-771.081C905.06,-753.813 1061.29,-716.086 1190,-739 1216.24,-743.671 1244.39,-754.072 1265.85,-763.228"/>
<polygon fill="black" stroke="black" points="1264.64,-766.519 1275.2,-767.323 1267.45,-760.107 1264.64,-766.519"/>
</g>
<!-- DDQN&#45;&gt;TD3 -->
<g id="edge13" class="edge"><title>DDQN&#45;&gt;TD3</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M841.665,-1465.89C845.208,-1463.01 848.754,-1459.99 852,-1457 924.2,-1390.55 914.851,-1340.09 1002,-1295 1095.57,-1246.59 1144.83,-1315.02 1237,-1264 1258.21,-1252.26 1275.29,-1230.52 1286.53,-1212.97"/>
<polygon fill="darkgray" stroke="darkgray" points="1289.64,-1214.61 1291.87,-1204.25 1283.67,-1210.95 1289.64,-1214.61"/>
<text text-anchor="middle" x="1044.5" y="-1298" font-family="sans-serif" font-size="10.00" fill="darkgray">double Q&#45;learning</text>
</g>
<!-- RAINBOW -->
<g id="node24" class="node"><title>RAINBOW</title>
<g id="a_node24"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#RAINBOW" xlink:title="Combines six DQN extensions, namely Double Q&#45;Learning, prioritized replay,
dueling networks, multi&#45;step learning, distributional DQN, and noisy DQN into
single model to achieve state of the art performance

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1173,-1568C1173,-1568 1122,-1568 1122,-1568 1116,-1568 1110,-1562 1110,-1556 1110,-1556 1110,-1544 1110,-1544 1110,-1538 1116,-1532 1122,-1532 1122,-1532 1173,-1532 1173,-1532 1179,-1532 1185,-1538 1185,-1544 1185,-1544 1185,-1556 1185,-1556 1185,-1562 1179,-1568 1173,-1568"/>
<text text-anchor="middle" x="1147.5" y="-1546.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">RAINBOW</text>
</a>
</g>
</g>
<!-- DDQN&#45;&gt;RAINBOW -->
<g id="edge31" class="edge"><title>DDQN&#45;&gt;RAINBOW</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M843.68,-1482.54C876.667,-1481.22 934.976,-1480.58 984,-1489 1031.38,-1497.13 1042.27,-1504.4 1087,-1522 1091.48,-1523.76 1096.1,-1525.7 1100.68,-1527.72"/>
<polygon fill="darkgray" stroke="darkgray" points="1099.45,-1531 1110.01,-1531.92 1102.33,-1524.62 1099.45,-1531"/>
</g>
<!-- Duelling&#45;DQN -->
<g id="node34" class="node"><title>Duelling&#45;DQN</title>
<g id="a_node34"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DuellingDQN" xlink:title="Duelling DQN represents two separate estimators: one for the state value
function and one for the state&#45;dependent action advantage function. The main
benefit of this factoring is to generalize learning across actions without
imposing any change to the underlying reinforcement learning algorithm.

(2016)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M972,-1534C972,-1534 901,-1534 901,-1534 895,-1534 889,-1528 889,-1522 889,-1522 889,-1510 889,-1510 889,-1504 895,-1498 901,-1498 901,-1498 972,-1498 972,-1498 978,-1498 984,-1504 984,-1510 984,-1510 984,-1522 984,-1522 984,-1528 978,-1534 972,-1534"/>
<text text-anchor="middle" x="936.5" y="-1512.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Duelling&#45;DQN</text>
</a>
</g>
</g>
<!-- DDQN&#45;&gt;Duelling&#45;DQN -->
<g id="edge30" class="edge"><title>DDQN&#45;&gt;Duelling&#45;DQN</title>
<path fill="none" stroke="black" d="M843.688,-1491.1C854.189,-1493.95 866.723,-1497.35 879.104,-1500.71"/>
<polygon fill="black" stroke="black" points="878.391,-1504.14 888.959,-1503.38 880.223,-1497.38 878.391,-1504.14"/>
</g>
<!-- DQN+HER&#45;&gt;DDPG+HER -->
<g id="edge14" class="edge"><title>DQN+HER&#45;&gt;DDPG+HER</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M1117.18,-1349.94C1112.46,-1345.9 1108.12,-1341.24 1105,-1336 1020.93,-1194.98 1027.55,-1132.92 1051.5,-970.5 1063.52,-888.987 1055.22,-858.653 1105,-793 1107.43,-789.796 1110.36,-786.872 1113.53,-784.23"/>
<text text-anchor="middle" x="1044.5" y="-973.5" font-family="sans-serif" font-size="10.00" fill="darkgray">HER</text>
</g>
<!-- APE&#45;X DQN&#45;&gt;APE&#45;X DDPG -->
<g id="edge15" class="edge"><title>APE&#45;X DQN&#45;&gt;APE&#45;X DDPG</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M1269.73,-1386.73C1263.92,-1382.03 1258.59,-1376.44 1255,-1370 1207.59,-1285.02 1210.7,-1030.37 1220,-933.5 1227.81,-852.158 1207.34,-819.378 1255,-753 1257.32,-749.764 1260.16,-746.863 1263.29,-744.274"/>
<text text-anchor="middle" x="1222.5" y="-936.5" font-family="sans-serif" font-size="10.00" fill="darkgray">APE&#45;X</text>
</g>
<!-- A3C&#45;&gt;ACER -->
<g id="edge58" class="edge"><title>A3C&#45;&gt;ACER</title>
<path fill="none" stroke="black" d="M963.547,-1115.48C1000.65,-1126.13 1068.84,-1145.7 1110.67,-1157.72"/>
<polygon fill="black" stroke="black" points="1109.88,-1161.13 1120.45,-1160.52 1111.81,-1154.4 1109.88,-1161.13"/>
</g>
<!-- A3C&#45;&gt;RAINBOW -->
<g id="edge16" class="edge"><title>A3C&#45;&gt;RAINBOW</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M963.583,-1125.18C970.521,-1130.24 977.802,-1136.04 984,-1142 1037.12,-1193.11 1060.29,-1203.3 1087,-1272 1107.26,-1324.12 1076.6,-1474.83 1105,-1523 1105.18,-1523.31 1105.37,-1523.61 1105.56,-1523.91"/>
<polygon fill="darkgray" stroke="darkgray" points="1103.09,-1526.42 1112.21,-1531.82 1108.45,-1521.91 1103.09,-1526.42"/>
</g>
<!-- A2C -->
<g id="node50" class="node"><title>A2C</title>
<g id="a_node50"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#A2C" xlink:title="A2C is a synchronous, deterministic variant of Asynchronous Advantage Actor
Critic (A3C). It uses multiple workers to avoid the use of a replay buffer.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-1114C1162.5,-1114 1132.5,-1114 1132.5,-1114 1126.5,-1114 1120.5,-1108 1120.5,-1102 1120.5,-1102 1120.5,-1090 1120.5,-1090 1120.5,-1084 1126.5,-1078 1132.5,-1078 1132.5,-1078 1162.5,-1078 1162.5,-1078 1168.5,-1078 1174.5,-1084 1174.5,-1090 1174.5,-1090 1174.5,-1102 1174.5,-1102 1174.5,-1108 1168.5,-1114 1162.5,-1114"/>
<text text-anchor="middle" x="1147.5" y="-1092.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">A2C</text>
</a>
</g>
</g>
<!-- A3C&#45;&gt;A2C -->
<g id="edge57" class="edge"><title>A3C&#45;&gt;A2C</title>
<path fill="none" stroke="black" d="M963.547,-1106.5C1000.57,-1104.38 1068.55,-1100.48 1110.41,-1098.07"/>
<polygon fill="black" stroke="black" points="1110.67,-1101.56 1120.45,-1097.5 1110.27,-1094.57 1110.67,-1101.56"/>
</g>
<!-- Q&#45;learning&#45;&gt;DQN -->
<g id="edge20" class="edge"><title>Q&#45;learning&#45;&gt;DQN</title>
<path fill="none" stroke="black" d="M553.764,-1341.37C611.535,-1345.04 721.999,-1352.06 779.186,-1355.69"/>
<polygon fill="black" stroke="black" points="779.018,-1359.19 789.22,-1356.33 779.462,-1352.2 779.018,-1359.19"/>
</g>
<!-- PER&#45;&gt;RAINBOW -->
<g id="edge32" class="edge"><title>PER&#45;&gt;RAINBOW</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M843.591,-1543.56C899.031,-1544.73 1029.23,-1547.5 1099.52,-1549"/>
<polygon fill="darkgray" stroke="darkgray" points="1099.78,-1552.51 1109.85,-1549.22 1099.93,-1545.51 1099.78,-1552.51"/>
</g>
<!-- QR&#45;DQN&#45;&gt;RAINBOW -->
<g id="edge34" class="edge"><title>QR&#45;DQN&#45;&gt;RAINBOW</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M1114.67,-1443.87C1090.9,-1447.15 1060.29,-1455.76 1044.5,-1478 1035.24,-1491.05 1035.24,-1500.95 1044.5,-1514 1057.17,-1531.85 1079.4,-1540.93 1099.97,-1545.5"/>
<polygon fill="darkgray" stroke="darkgray" points="1099.4,-1548.96 1109.88,-1547.41 1100.73,-1542.09 1099.4,-1548.96"/>
</g>
<!-- NGU -->
<g id="node35" class="node"><title>NGU</title>
<g id="a_node35"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#NGU" xlink:title="Never Give Up (NGU). (from the abstract) We propose a reinforcement learning
agent to solve hard exploration games by learning a range of directed
exploratory policies. We construct an episodic memory&#45;based intrinsic reward
using k&#45;nearest neighbors over the agent&#39;s recent experience to train the
directed exploratory policies, thereby encouraging the agent to repeatedly
revisit all states in its environment. A self&#45;supervised inverse dynamics
model is used to train the embeddings of the nearest neighbour lookup, biasing
the novelty signal towards what the agent can control. We employ the framework
of Universal Value Function Approximators (UVFA) to simultaneously learn many
directed exploration policies with the same neural network, with different
trade&#45;offs between exploration and exploitation. By using the same neural
network for different degrees of exploration/exploitation, transfer is
demonstrated from predominantly exploratory policies yielding effective
exploitative policies. The proposed method can be incorporated to run with
modern distributed RL agents that collect large amounts of experience from
many actors running in parallel on separate environment instances. Our method
doubles the performance of the base agent in all hard exploration in the
Atari&#45;57 suite while maintaining a very high score across the remaining games,
obtaining a median human normalised score of 1344.0%. Notably, the proposed
method is the first algorithm to achieve non&#45;zero rewards (with a mean score
of 8,400) in the game of Pitfall! without using demonstrations or hand&#45;crafted
features.

(2020)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1532,-1383C1532,-1383 1502,-1383 1502,-1383 1496,-1383 1490,-1377 1490,-1371 1490,-1371 1490,-1359 1490,-1359 1490,-1353 1496,-1347 1502,-1347 1502,-1347 1532,-1347 1532,-1347 1538,-1347 1544,-1353 1544,-1359 1544,-1359 1544,-1371 1544,-1371 1544,-1377 1538,-1383 1532,-1383"/>
<text text-anchor="middle" x="1517" y="-1361.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">NGU</text>
</a>
</g>
</g>
<!-- R2D2&#45;&gt;NGU -->
<g id="edge35" class="edge"><title>R2D2&#45;&gt;NGU</title>
<path fill="none" stroke="black" d="M1444.63,-1365C1455.34,-1365 1467.92,-1365 1479.51,-1365"/>
<polygon fill="black" stroke="black" points="1479.8,-1368.5 1489.8,-1365 1479.8,-1361.5 1479.8,-1368.5"/>
</g>
<!-- Duelling&#45;DQN&#45;&gt;RAINBOW -->
<g id="edge33" class="edge"><title>Duelling&#45;DQN&#45;&gt;RAINBOW</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M984.144,-1523.59C1018.49,-1529.17 1065.01,-1536.74 1099.45,-1542.35"/>
<polygon fill="darkgray" stroke="darkgray" points="1099.29,-1545.87 1109.72,-1544.02 1100.41,-1538.96 1099.29,-1545.87"/>
</g>
<!-- Agent57 -->
<g id="node36" class="node"><title>Agent57</title>
<g id="a_node36"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#Agent57" xlink:title="(from the abstract) Atari games have been a long&#45;standing benchmark in the
reinforcement learning (RL) community for the past decade. This benchmark was
proposed to test general competency of RL algorithms. Previous work has
achieved good average performance by doing outstandingly well on many games of
the set, but very poorly in several of the most challenging games. We propose
Agent57, the first deep RL agent that outperforms the standard human benchmark
on all 57 Atari games. To achieve this result, we train a neural network which
parameterizes a family of policies ranging from very exploratory to purely
exploitative. We propose an adaptive mechanism to choose which policy to
prioritize throughout the training process. Additionally, we utilize a novel
parameterization of the architecture that allows for more consistent and
stable learning.

(2020)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1537,-1455C1537,-1455 1497,-1455 1497,-1455 1491,-1455 1485,-1449 1485,-1443 1485,-1443 1485,-1431 1485,-1431 1485,-1425 1491,-1419 1497,-1419 1497,-1419 1537,-1419 1537,-1419 1543,-1419 1549,-1425 1549,-1431 1549,-1431 1549,-1443 1549,-1443 1549,-1449 1543,-1455 1537,-1455"/>
<text text-anchor="middle" x="1517" y="-1433.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Agent57</text>
</a>
</g>
</g>
<!-- NGU&#45;&gt;Agent57 -->
<g id="edge36" class="edge"><title>NGU&#45;&gt;Agent57</title>
<path fill="none" stroke="black" d="M1517,-1383.28C1517,-1391.83 1517,-1400.37 1517,-1408.92"/>
<polygon fill="black" stroke="black" points="1513.5,-1408.95 1517,-1418.95 1520.5,-1408.95 1513.5,-1408.95"/>
</g>
<!-- DPG&#45;&gt;DDPG -->
<g id="edge48" class="edge"><title>DPG&#45;&gt;DDPG</title>
<path fill="none" stroke="black" d="M816.5,-1002.66C816.5,-937.592 816.5,-872.523 816.5,-807.454"/>
<polygon fill="black" stroke="black" points="820,-807.434 816.5,-797.434 813,-807.434 820,-807.434"/>
</g>
<!-- TRPO&#45;&gt;ACER -->
<g id="edge55" class="edge"><title>TRPO&#45;&gt;ACER</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M843.762,-1123.33C857.183,-1127.34 873.841,-1131.93 889,-1135 967.146,-1150.85 1060.26,-1160.55 1110.33,-1165.04"/>
<polygon fill="darkgray" stroke="darkgray" points="1110.08,-1168.53 1120.34,-1165.91 1110.69,-1161.55 1110.08,-1168.53"/>
<text text-anchor="middle" x="936.5" y="-1154" font-family="sans-serif" font-size="10.00" fill="darkgray">TRPO technique</text>
</g>
<!-- TRPO&#45;&gt;GAE -->
<g id="edge54" class="edge"><title>TRPO&#45;&gt;GAE</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M816.5,-1133.28C816.5,-1141.83 816.5,-1150.37 816.5,-1158.92"/>
<polygon fill="darkgray" stroke="darkgray" points="813,-1158.95 816.5,-1168.95 820,-1158.95 813,-1158.95"/>
</g>
<!-- PPO -->
<g id="node49" class="node"><title>PPO</title>
<g id="a_node49"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PPO" xlink:title="Proximal Policy Optimization (PPO) is similar to TRPO but uses simpler
mechanism while retaining similar performance.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-1240C1162.5,-1240 1132.5,-1240 1132.5,-1240 1126.5,-1240 1120.5,-1234 1120.5,-1228 1120.5,-1228 1120.5,-1216 1120.5,-1216 1120.5,-1210 1126.5,-1204 1132.5,-1204 1132.5,-1204 1162.5,-1204 1162.5,-1204 1168.5,-1204 1174.5,-1210 1174.5,-1216 1174.5,-1216 1174.5,-1228 1174.5,-1228 1174.5,-1234 1168.5,-1240 1162.5,-1240"/>
<text text-anchor="middle" x="1147.5" y="-1218.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">PPO</text>
</a>
</g>
</g>
<!-- TRPO&#45;&gt;PPO -->
<g id="edge56" class="edge"><title>TRPO&#45;&gt;PPO</title>
<path fill="none" stroke="darkgray" stroke-dasharray="5,2" d="M838.01,-1133.07C851.604,-1144.14 870.318,-1157.73 889,-1166 963.398,-1198.94 1058.7,-1213.14 1109.96,-1218.74"/>
<polygon fill="darkgray" stroke="darkgray" points="1109.9,-1222.25 1120.21,-1219.81 1110.62,-1215.29 1109.9,-1222.25"/>
</g>
<!-- PPO&#45;&gt;SAC -->
<!-- A2C&#45;&gt;ACER -->
<!-- A2C&#45;&gt;ACKTR -->
<!-- A2C&#45;&gt;SVPG -->
<!-- A2C&#45;&gt;IMPALA -->
<!-- Dyna&#45;Q -->
<g id="node52" class="node"><title>Dyna&#45;Q</title>
<g id="a_node52"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DynaQ" xlink:title="Dyna&#45;Q uses the experience drawn from real interaction with the environment to
improve the value function/policy (called direct RL, using Q&#45;learning) and the
model of the environment (called model learning). The model is then used to
create experiences (called planning) to improve the value function/policy.

(1990)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M533,-518C533,-518 498,-518 498,-518 492,-518 486,-512 486,-506 486,-506 486,-494 486,-494 486,-488 492,-482 498,-482 498,-482 533,-482 533,-482 539,-482 545,-488 545,-494 545,-494 545,-506 545,-506 545,-512 539,-518 533,-518"/>
<text text-anchor="middle" x="515.5" y="-496.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Dyna&#45;Q</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;Dyna&#45;Q -->
<g id="edge64" class="edge"><title>Model Based&#45;&gt;Dyna&#45;Q</title>
<path fill="none" stroke="black" d="M235.28,-411.163C245.817,-428.806 264.308,-454.804 288,-468 347.349,-501.058 428.517,-504.225 475.844,-502.687"/>
<polygon fill="black" stroke="black" points="476.131,-506.178 485.978,-502.268 475.842,-499.184 476.131,-506.178"/>
</g>
<!-- MCTS -->
<g id="node53" class="node"><title>MCTS</title>
<g id="a_node53"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MCTS" xlink:title="Monte Carlo Tree Search (MCTS) selects the next action by performing rollout
algorithm, which estimates action values for a given policy by averaging the
returns of many simulated trajectories that start with each possible action
and then follow the given policy. Unlike Monte Carlo control, the goal of a
rollout algorithm is not to estimate a complete optimal action&#45;value function,
q&#45;star, or a complete action&#45;value function,q&#45;pi, for a given policy pi.
Instead, they produce Monte Carlo estimates of action values only for each
current state, and once an action is selected, this estimation will be
discarded and fresh calculation will be performed on the next state. MCTS
enchances this rollout algorithm by the addition of a means for accumulating
value estimates obtained from the Monte Carlo simulations in order to
successively direct simulations toward more highly&#45;rewarding trajectories.

(2006)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M667,-610C667,-610 637,-610 637,-610 631,-610 625,-604 625,-598 625,-598 625,-586 625,-586 625,-580 631,-574 637,-574 637,-574 667,-574 667,-574 673,-574 679,-580 679,-586 679,-586 679,-598 679,-598 679,-604 673,-610 667,-610"/>
<text text-anchor="middle" x="652" y="-588.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MCTS</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;MCTS -->
<g id="edge65" class="edge"><title>Model Based&#45;&gt;MCTS</title>
<path fill="none" stroke="black" d="M236.441,-411.323C249.614,-432.473 273.207,-467.603 299,-493 356.91,-550.02 370.583,-574.272 448,-599 504.206,-616.953 573.659,-608.448 615.032,-600.401"/>
<polygon fill="black" stroke="black" points="615.823,-603.812 624.921,-598.381 614.422,-596.953 615.823,-603.812"/>
</g>
<!-- PILCO -->
<g id="node54" class="node"><title>PILCO</title>
<g id="a_node54"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PILCO" xlink:title="(from the abstract) In this paper, we introduce PILCO, a practical, data&#45;
efficient model&#45;based policy search method. PILCO reduces model bias, one of
the key problems of model&#45;based reinforcement learning, in a principled way.
By learning a probabilistic dynamics model and explicitly incorporating model
uncertainty into long&#45;term planning, PILCO can cope with very little data and
facilitates learning froms cratch in only a few trials. Policy evaluationis
performed in closed form using state&#45;of&#45;the&#45;art approximate inference.
Furthermore, policy gradients are computed analytically for policy
improvement. We report unprecedented learning efficiency on challenging and
high&#45;dimensional control tasks.

(2011)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M831.5,-630C831.5,-630 801.5,-630 801.5,-630 795.5,-630 789.5,-624 789.5,-618 789.5,-618 789.5,-606 789.5,-606 789.5,-600 795.5,-594 801.5,-594 801.5,-594 831.5,-594 831.5,-594 837.5,-594 843.5,-600 843.5,-606 843.5,-606 843.5,-618 843.5,-618 843.5,-624 837.5,-630 831.5,-630"/>
<text text-anchor="middle" x="816.5" y="-608.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">PILCO</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;PILCO -->
<g id="edge66" class="edge"><title>Model Based&#45;&gt;PILCO</title>
<path fill="none" stroke="black" d="M227.028,-411.127C232.77,-466.389 258.15,-629 358,-629 358,-629 358,-629 653,-629 696.641,-629 746.518,-622.86 779.374,-617.96"/>
<polygon fill="black" stroke="black" points="780.059,-621.396 789.413,-616.422 778.999,-614.477 780.059,-621.396"/>
</g>
<!-- I2A -->
<g id="node55" class="node"><title>I2A</title>
<g id="a_node55"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#I2A" xlink:title="(from the abstract) We introduce Imagination&#45;Augmented Agents (I2As), a novel
architecture for deep reinforcement learning combining model&#45;free and model&#45;
based aspects. In contrast to most existing model&#45;based reinforcement learning
and planning methods, which prescribe how a model should be used to arrive at
a policy, I2As learn to interpret predictions from a learned environment model
to construct implicit plans in arbitrary ways, by using the predictions as
additional context in deep policy networks. I2As show improved data
efficiency, performance, and robustness to model misspecification compared to
several baselines.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-650C1162.5,-650 1132.5,-650 1132.5,-650 1126.5,-650 1120.5,-644 1120.5,-638 1120.5,-638 1120.5,-626 1120.5,-626 1120.5,-620 1126.5,-614 1132.5,-614 1132.5,-614 1162.5,-614 1162.5,-614 1168.5,-614 1174.5,-620 1174.5,-626 1174.5,-626 1174.5,-638 1174.5,-638 1174.5,-644 1168.5,-650 1162.5,-650"/>
<text text-anchor="middle" x="1147.5" y="-628.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">I2A</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;I2A -->
<g id="edge67" class="edge"><title>Model Based&#45;&gt;I2A</title>
<path fill="none" stroke="black" d="M226.354,-411.001C230.173,-469.272 250.654,-649 358,-649 358,-649 358,-649 937.5,-649 998.283,-649 1068.68,-641.769 1110.3,-636.739"/>
<polygon fill="black" stroke="black" points="1110.99,-640.18 1120.49,-635.481 1110.14,-633.232 1110.99,-640.18"/>
</g>
<!-- MBMF -->
<g id="node56" class="node"><title>MBMF</title>
<g id="a_node56"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MBMF" xlink:title="(from the abstract) Neural Network Dynamics for Model&#45;Based Deep Reinforcement
Learning with Model&#45;Free Fine&#45;Tuning. We demonstrate that medium&#45;sized neural
network models can in fact be combined with model predictive control (MPC) to
achieve excellent sample complexity in a model&#45;based reinforcement learning
algorithm, producing stable and plausible gaits to accomplish various complex
locomotion tasks. We also propose using deep neural network dynamics models to
initialize a model&#45;free learner, in order to combine the sample efficiency of
model&#45;based approaches with the high task&#45;specific performance of model&#45;free
methods. We empirically demonstrate on MuJoCo locomotion tasks that our pure
model&#45;based approach trained on just random action data can follow arbitrary
trajectories with excellent sample efficiency, and that our hybrid algorithm
can accelerate model&#45;free learning on high&#45;speed benchmark tasks, achieving
sample efficiency gains of 3&#45;5x on swimmer, cheetah, hopper, and ant agents.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-278C1162.5,-278 1132.5,-278 1132.5,-278 1126.5,-278 1120.5,-272 1120.5,-266 1120.5,-266 1120.5,-254 1120.5,-254 1120.5,-248 1126.5,-242 1132.5,-242 1132.5,-242 1162.5,-242 1162.5,-242 1168.5,-242 1174.5,-248 1174.5,-254 1174.5,-254 1174.5,-266 1174.5,-266 1174.5,-272 1168.5,-278 1162.5,-278"/>
<text text-anchor="middle" x="1147.5" y="-256.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MBMF</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;MBMF -->
<g id="edge68" class="edge"><title>Model Based&#45;&gt;MBMF</title>
<path fill="none" stroke="black" d="M230.425,-374.803C239.223,-345.822 260.432,-289.365 299,-259 320.347,-242.193 330.831,-243 358,-243 358,-243 358,-243 937.5,-243 998.283,-243 1068.68,-250.231 1110.3,-255.261"/>
<polygon fill="black" stroke="black" points="1110.14,-258.768 1120.49,-256.519 1110.99,-251.82 1110.14,-258.768"/>
</g>
<!-- Exit -->
<g id="node57" class="node"><title>Exit</title>
<g id="a_node57"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#Exit" xlink:title="Expert Iteration (ExIt) is a novel reinforcement learning algorithm which
decomposes the problem into separate planning and generalisation tasks.
Planning new policies is performed by tree search, while a deep neural network
generalises those plans. Subsequently, tree search is improved by using the
neural network policy to guide search, increasing the strength of new plans.
In contrast, standard deep Reinforcement Learning algorithms rely on a neural
network not only to generalise plans, but to discover them too. We show that
ExIt outperforms REINFORCE for training a neural network to play the board
game Hex, and our final tree search agent, trained tabula rasa, defeats MoHex
1.0, the most recent Olympiad Champion player to be publicly released. (from
the abstract)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-332C1162.5,-332 1132.5,-332 1132.5,-332 1126.5,-332 1120.5,-326 1120.5,-320 1120.5,-320 1120.5,-308 1120.5,-308 1120.5,-302 1126.5,-296 1132.5,-296 1132.5,-296 1162.5,-296 1162.5,-296 1168.5,-296 1174.5,-302 1174.5,-308 1174.5,-308 1174.5,-320 1174.5,-320 1174.5,-326 1168.5,-332 1162.5,-332"/>
<text text-anchor="middle" x="1147.5" y="-310.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Exit</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;Exit -->
<g id="edge69" class="edge"><title>Model Based&#45;&gt;Exit</title>
<path fill="none" stroke="black" d="M232.015,-374.737C242.158,-349.083 264.245,-302.851 299,-279 321.402,-263.627 330.831,-263 358,-263 358,-263 358,-263 937.5,-263 999.78,-263 1069.6,-284.638 1110.71,-299.726"/>
<polygon fill="black" stroke="black" points="1109.69,-303.078 1120.28,-303.309 1112.14,-296.523 1109.69,-303.078"/>
</g>
<!-- AlphaZero -->
<g id="node58" class="node"><title>AlphaZero</title>
<g id="a_node58"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#AlphaZero" xlink:title="AlphaZero generalises tabula rasa reinforcement learning from games of self&#45;
play approach. Starting from random play, and given no domain knowledge except
the game rules, AlphaZero achieved within 24 hours a superhuman level of play
in the games of chess and shogi (Japanese chess) as well as Go, and
convincingly defeated a world&#45;champion program in each case. (from the
abstract)

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1173,-386C1173,-386 1122,-386 1122,-386 1116,-386 1110,-380 1110,-374 1110,-374 1110,-362 1110,-362 1110,-356 1116,-350 1122,-350 1122,-350 1173,-350 1173,-350 1179,-350 1185,-356 1185,-362 1185,-362 1185,-374 1185,-374 1185,-380 1179,-386 1173,-386"/>
<text text-anchor="middle" x="1147.5" y="-364.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">AlphaZero</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;AlphaZero -->
<g id="edge70" class="edge"><title>Model Based&#45;&gt;AlphaZero</title>
<path fill="none" stroke="black" d="M232.38,-374.84C241.611,-353.045 260.137,-317.062 288,-298 314.26,-280.035 326.183,-283 358,-283 358,-283 358,-283 937.5,-283 970.688,-283 1051.51,-320.242 1102.37,-345.424"/>
<polygon fill="black" stroke="black" points="1100.99,-348.645 1111.5,-349.974 1104.11,-342.38 1100.99,-348.645"/>
</g>
<!-- MVE -->
<g id="node59" class="node"><title>MVE</title>
<g id="a_node59"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MVE" xlink:title="(from the abstract) Recent model&#45;free reinforcement learning algorithms have
proposed incorporating learned dynamics models as a source of additional data
with the intention of reducing sample complexity. Such methods hold the
promise of incorporating imagined data coupled with a notion of model
uncertainty to accelerate the learning of continuous control tasks.
Unfortunately, they rely on heuristics that limit usage of the dynamics model.
We present model&#45;based value expansion, which controls for uncertainty in the
model by only allowing imagination to fixed depth. By enabling wider use of
learned dynamics models within a model&#45;free reinforcement learning algorithm,
we improve value estimation, which, in turn, reduces the sample complexity of
learning.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-278C1317.5,-278 1287.5,-278 1287.5,-278 1281.5,-278 1275.5,-272 1275.5,-266 1275.5,-266 1275.5,-254 1275.5,-254 1275.5,-248 1281.5,-242 1287.5,-242 1287.5,-242 1317.5,-242 1317.5,-242 1323.5,-242 1329.5,-248 1329.5,-254 1329.5,-254 1329.5,-266 1329.5,-266 1329.5,-272 1323.5,-278 1317.5,-278"/>
<text text-anchor="middle" x="1302.5" y="-256.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MVE</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;MVE -->
<g id="edge71" class="edge"><title>Model Based&#45;&gt;MVE</title>
<path fill="none" stroke="black" d="M236.938,-374.717C249.571,-356.435 271.771,-329.44 299,-318 344.892,-298.719 361.226,-317.421 411,-318 487.46,-318.889 506.655,-317.715 583,-322 816.89,-335.126 871.409,-377.346 1105,-395 1142.67,-397.847 1157.11,-413.591 1190,-395 1238.77,-367.431 1215.08,-326.302 1255,-287 1258.48,-283.578 1262.47,-280.428 1266.63,-277.582"/>
<polygon fill="black" stroke="black" points="1268.65,-280.446 1275.27,-272.176 1264.94,-274.511 1268.65,-280.446"/>
</g>
<!-- STEVE -->
<g id="node60" class="node"><title>STEVE</title>
<g id="a_node60"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#STEVE" xlink:title="(from the abstract) Integrating model&#45;free and model&#45;based approaches in
reinforcement learning has the potential to achieve the high performance of
model&#45;free algorithms with low sample complexity. However, this is difficult
because an imperfect dynamics model can degrade the performance of the
learning algorithm, and in sufficiently complex environments, the dynamics
model will almost always be imperfect. As a result, a key challenge is to
combine model&#45;based approaches with model&#45;free learning in such a way that
errors in the model do not degrade performance. We propose stochastic ensemble
value expansion (STEVE), a novel model&#45;based technique that addresses this
issue. By dynamically interpolating between model rollouts of various horizon
lengths for each individual example, STEVE ensures that the model is only
utilized when doing so does not introduce significant errors. Our approach
outperforms model&#45;free baselines on challenging continuous control benchmarks
with an order&#45;of&#45;magnitude increase in sample efficiency, and in contrast to
previous model&#45;based approaches, performance does not degrade in complex
environments.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1319,-332C1319,-332 1286,-332 1286,-332 1280,-332 1274,-326 1274,-320 1274,-320 1274,-308 1274,-308 1274,-302 1280,-296 1286,-296 1286,-296 1319,-296 1319,-296 1325,-296 1331,-302 1331,-308 1331,-308 1331,-320 1331,-320 1331,-326 1325,-332 1319,-332"/>
<text text-anchor="middle" x="1302.5" y="-310.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">STEVE</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;STEVE -->
<g id="edge72" class="edge"><title>Model Based&#45;&gt;STEVE</title>
<path fill="none" stroke="black" d="M240.252,-374.974C253.751,-360.008 275.2,-339.801 299,-331 353.193,-310.959 371.221,-330.222 429,-330 497.444,-329.737 515.14,-321.075 583,-330 673.32,-341.879 691.251,-367.377 781,-383 923.358,-407.78 960.626,-403.995 1105,-410 1163.62,-412.438 1190.58,-445.868 1237,-410 1262.08,-390.623 1235.82,-366.231 1255,-341 1257.81,-337.305 1261.28,-334.027 1265.05,-331.143"/>
<polygon fill="black" stroke="black" points="1267.24,-333.891 1273.64,-325.451 1263.37,-328.055 1267.24,-333.891"/>
</g>
<!-- ME&#45;TRPO -->
<g id="node61" class="node"><title>ME&#45;TRPO</title>
<g id="a_node61"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#METRPO" xlink:title="(from the abstract) Model&#45;free reinforcement learning (RL) methods are
succeeding in a growing number of tasks, aided by recent advances in deep
learning. However, they tend to suffer from high sample complexity, which
hinders their use in real&#45;world domains. Alternatively, model&#45;based
reinforcement learning promises to reduce sample complexity, but tends to
require careful tuning and to date have succeeded mainly in restrictive
domains where simple models are sufficient for learning. In this paper, we
analyze the behavior of vanilla model&#45;based reinforcement learning methods
when deep neural networks are used to learn both the model and the policy, and
show that the learned policy tends to exploit regions where insufficient data
is available for the model to be learned, causing instability in training. To
overcome this issue, we propose to use an ensemble of models to maintain the
model uncertainty and regularize the learning process. We further show that
the use of likelihood ratio derivatives yields much more stable learning than
backpropagation through time. Altogether, our approach Model&#45;Ensemble Trust&#45;
Region Policy Optimization (ME&#45;TRPO) significantly reduces the sample
complexity compared to model&#45;free deep RL methods on challenging continuous
control benchmark tasks.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1326.5,-386C1326.5,-386 1278.5,-386 1278.5,-386 1272.5,-386 1266.5,-380 1266.5,-374 1266.5,-374 1266.5,-362 1266.5,-362 1266.5,-356 1272.5,-350 1278.5,-350 1278.5,-350 1326.5,-350 1326.5,-350 1332.5,-350 1338.5,-356 1338.5,-362 1338.5,-362 1338.5,-374 1338.5,-374 1338.5,-380 1332.5,-386 1326.5,-386"/>
<text text-anchor="middle" x="1302.5" y="-364.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">ME&#45;TRPO</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;ME&#45;TRPO -->
<g id="edge73" class="edge"><title>Model Based&#45;&gt;ME&#45;TRPO</title>
<path fill="none" stroke="black" d="M250.984,-374.832C261.85,-368.15 275.004,-361.391 288,-358 414.863,-324.897 456.033,-325.298 583,-358 597.255,-361.672 598.179,-368.934 612,-374 683.936,-400.369 705.062,-397.828 781,-408 924.054,-427.163 960.751,-425.108 1105,-430 1163.63,-431.988 1186.92,-460.559 1237,-430 1251.93,-420.889 1242.95,-407.681 1255,-395 1256.04,-393.9 1257.14,-392.831 1258.29,-391.794"/>
<polygon fill="black" stroke="black" points="1260.68,-394.361 1266.35,-385.407 1256.34,-388.875 1260.68,-394.361"/>
</g>
<!-- MB&#45;MPO -->
<g id="node62" class="node"><title>MB&#45;MPO</title>
<g id="a_node62"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MBMPO" xlink:title="(from the abstract) Model&#45;based reinforcement learning approaches carry the
promise of being data efficient. However, due to challenges in learning
dynamics models that sufficiently match the real&#45;world dynamics, they struggle
to achieve the same asymptotic performance as model&#45;free methods. We propose
Model&#45;Based Meta&#45;Policy&#45;Optimization (MB&#45;MPO), an approach that foregoes the
strong reliance on accurate learned dynamics models. Using an ensemble of
learned dynamic models, MB&#45;MPO meta&#45;learns a policy that can quickly adapt to
any model in the ensemble with one policy gradient step. This steers the meta&#45;
policy towards internalizing consistent dynamics predictions among the
ensemble while shifting the burden of behaving optimally w.r.t. the model
discrepancies towards the adaptation step. Our experiments show that MB&#45;MPO is
more robust to model imperfections than previous model&#45;based approaches.
Finally, we demonstrate that our approach is able to match the asymptotic
performance of model&#45;free methods while requiring significantly less
experience.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1323.5,-440C1323.5,-440 1281.5,-440 1281.5,-440 1275.5,-440 1269.5,-434 1269.5,-428 1269.5,-428 1269.5,-416 1269.5,-416 1269.5,-410 1275.5,-404 1281.5,-404 1281.5,-404 1323.5,-404 1323.5,-404 1329.5,-404 1335.5,-410 1335.5,-416 1335.5,-416 1335.5,-428 1335.5,-428 1335.5,-434 1329.5,-440 1323.5,-440"/>
<text text-anchor="middle" x="1302.5" y="-418.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MB&#45;MPO</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;MB&#45;MPO -->
<g id="edge74" class="edge"><title>Model Based&#45;&gt;MB&#45;MPO</title>
<path fill="none" stroke="black" d="M270.334,-380.904C276.237,-379.706 282.238,-378.682 288,-378 418.203,-362.594 454.404,-352.445 583,-378 596.681,-380.719 598.632,-386.018 612,-390 651.013,-401.621 661.829,-400.413 702,-407 879.416,-436.09 925.296,-436.597 1105,-442 1163.64,-443.763 1179,-450.787 1237,-442 1244.5,-440.865 1252.32,-439.027 1259.83,-436.914"/>
<polygon fill="black" stroke="black" points="1260.9,-440.247 1269.46,-434.013 1258.88,-433.544 1260.9,-440.247"/>
</g>
<!-- World Models -->
<g id="node63" class="node"><title>World Models</title>
<g id="a_node63"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#WorldModels" xlink:title="(from the abstract) A generative recurrent neural network is quickly trained
in an unsupervised manner to model popular reinforcement learning environments
through compressed spatio&#45;temporal representations. The world model&#39;s
extracted features are fed into compact and simple policies trained by
evolution, achieving state of the art results in various environments. We also
train our agent entirely inside of an environment generated by its own
internal world model, and transfer this policy back into the actual
environment.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1338,-494C1338,-494 1267,-494 1267,-494 1261,-494 1255,-488 1255,-482 1255,-482 1255,-470 1255,-470 1255,-464 1261,-458 1267,-458 1267,-458 1338,-458 1338,-458 1344,-458 1350,-464 1350,-470 1350,-470 1350,-482 1350,-482 1350,-488 1344,-494 1338,-494"/>
<text text-anchor="middle" x="1302.5" y="-472.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">World Models</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;World Models -->
<g id="edge75" class="edge"><title>Model Based&#45;&gt;World Models</title>
<path fill="none" stroke="black" d="M270.206,-391.384C338.007,-389.439 470.819,-387.609 583,-398 596.023,-399.206 599.093,-400.884 612,-403 651.948,-409.55 661.989,-410.849 702,-417 737.102,-422.396 745.789,-424.371 781,-429 948.95,-451.077 1148.76,-465.951 1244.64,-472.385"/>
<polygon fill="black" stroke="black" points="1244.49,-475.883 1254.7,-473.055 1244.96,-468.899 1244.49,-475.883"/>
</g>
<!-- PETS -->
<g id="node64" class="node"><title>PETS</title>
<g id="a_node64"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PETS" xlink:title="(from the abstract) Model&#45;based reinforcement learning (RL) algorithms can
attain excellent sample efficiency, but often lag behind the best model&#45;free
algorithms in terms of asymptotic performance. This is especially true with
high&#45;capacity parametric function approximators, such as deep networks. In
this paper, we study how to bridge this gap, by employing uncertainty&#45;aware
dynamics models. We propose a new algorithm called probabilistic ensembles
with trajectory sampling (PETS) that combines uncertainty&#45;aware deep network
dynamics models with sampling&#45;based uncertainty propagation. Our comparison to
state&#45;of&#45;the&#45;art model&#45;based and model&#45;free deep RL algorithms shows that our
approach matches the asymptotic performance of model&#45;free algorithms on
several challenging benchmark tasks, while requiring significantly fewer
samples (e.g., 8 and 125 times fewer samples than Soft Actor Critic and
Proximal Policy Optimization respectively on the half&#45;cheetah task).

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-548C1317.5,-548 1287.5,-548 1287.5,-548 1281.5,-548 1275.5,-542 1275.5,-536 1275.5,-536 1275.5,-524 1275.5,-524 1275.5,-518 1281.5,-512 1287.5,-512 1287.5,-512 1317.5,-512 1317.5,-512 1323.5,-512 1329.5,-518 1329.5,-524 1329.5,-524 1329.5,-536 1329.5,-536 1329.5,-542 1323.5,-548 1317.5,-548"/>
<text text-anchor="middle" x="1302.5" y="-526.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">PETS</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;PETS -->
<g id="edge76" class="edge"><title>Model Based&#45;&gt;PETS</title>
<path fill="none" stroke="black" d="M270.231,-395.652C352.23,-400.593 523.647,-411.311 583,-418 845.888,-447.627 1159.83,-503.797 1265.09,-523.203"/>
<polygon fill="black" stroke="black" points="1264.72,-526.693 1275.19,-525.07 1265.99,-519.81 1264.72,-526.693"/>
</g>
<!-- PlaNet -->
<g id="node65" class="node"><title>PlaNet</title>
<g id="a_node65"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PlaNet" xlink:title="(from the abstract) We propose the Deep Planning Network (PlaNet), a purely
model&#45;based agent that learns the environment dynamics from images and chooses
actions through fast online planning in latent space. To achieve high
performance, the dynamics model must accurately predict the rewards ahead for
multiple time steps. We approach this using a latent dynamics model with both
deterministic and stochastic transition components. Moreover, we propose a
multi&#45;step variational inference objective that we name latent overshooting.
Using only pixel observations, our agent solves continuous control tasks with
contact dynamics, partial observability, and sparse rewards, which exceed the
difficulty of tasks that were previously solved by planning with learned
models. PlaNet uses substantially fewer episodes and reaches final performance
close to and sometimes higher than strong model&#45;free algorithms.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1317.5,-602C1317.5,-602 1287.5,-602 1287.5,-602 1281.5,-602 1275.5,-596 1275.5,-590 1275.5,-590 1275.5,-578 1275.5,-578 1275.5,-572 1281.5,-566 1287.5,-566 1287.5,-566 1317.5,-566 1317.5,-566 1323.5,-566 1329.5,-572 1329.5,-578 1329.5,-578 1329.5,-590 1329.5,-590 1329.5,-596 1323.5,-602 1317.5,-602"/>
<text text-anchor="middle" x="1302.5" y="-580.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">PlaNet</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;PlaNet -->
<g id="edge77" class="edge"><title>Model Based&#45;&gt;PlaNet</title>
<path fill="none" stroke="black" d="M270.126,-401.356C279.638,-403.007 289.638,-404.646 299,-406 424.713,-424.182 457.523,-418.257 583,-438 846.762,-479.502 1160.68,-551.016 1265.47,-575.494"/>
<polygon fill="black" stroke="black" points="1264.7,-578.908 1275.23,-577.781 1266.29,-572.093 1264.7,-578.908"/>
</g>
<!-- SimPLe -->
<g id="node66" class="node"><title>SimPLe</title>
<g id="a_node66"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#SimPLe" xlink:title="Simulated Policy Learning (SimPLe) is a complete model&#45;based deep RL algorithm
based on video prediction models and present a comparison of several model
architectures, including a novel architecture that yields the best results in
our setting. Our experiments evaluate SimPLe on a range of Atari games in low
data regime of 100k interactions between the agent and the environment, which
corresponds to two hours of real&#45;time play. In most games SimPLe outperforms
state&#45;of&#45;the&#45;art model&#45;free algorithms, in some games by over an order of
magnitude. (from the abstract)

(2019)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1436,-596C1436,-596 1399,-596 1399,-596 1393,-596 1387,-590 1387,-584 1387,-584 1387,-572 1387,-572 1387,-566 1393,-560 1399,-560 1399,-560 1436,-560 1436,-560 1442,-560 1448,-566 1448,-572 1448,-572 1448,-584 1448,-584 1448,-590 1442,-596 1436,-596"/>
<text text-anchor="middle" x="1417.5" y="-574.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">SimPLe</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;SimPLe -->
<g id="edge78" class="edge"><title>Model Based&#45;&gt;SimPLe</title>
<path fill="none" stroke="black" d="M270.198,-409.694C279.616,-412.789 289.56,-415.751 299,-418 422.997,-447.543 457.471,-435.849 583,-458 854.606,-505.928 925.057,-510.369 1190,-587 1219.58,-595.556 1224.69,-605.582 1255,-611 1299.41,-618.936 1312.81,-622.223 1356.5,-611 1365.02,-608.81 1373.64,-605.128 1381.57,-600.998"/>
<polygon fill="black" stroke="black" points="1383.5,-603.932 1390.53,-596.008 1380.09,-597.816 1383.5,-603.932"/>
</g>
<!-- MuZero -->
<g id="node67" class="node"><title>MuZero</title>
<g id="a_node67"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MuZero" xlink:title="(from the abstract) Constructing agents with planning capabilities has long
been one of the main challenges in the pursuit of artificial intelligence.
Tree&#45;based planning methods have enjoyed huge success in challenging domains,
such as chess and Go, where a perfect simulator is available. However, in
real&#45;world problems the dynamics governing the environment are often complex
and unknown. In this work we present the MuZero algorithm which, by combining
a tree&#45;based search with a learned model, achieves superhuman performance in a
range of challenging and visually complex domains, without any knowledge of
their underlying dynamics. MuZero learns a model that, when applied
iteratively, predicts the quantities most directly relevant to planning: the
reward, the action&#45;selection policy, and the value function. When evaluated on
57 different Atari games &#45; the canonical video game environment for testing AI
techniques, in which model&#45;based planning approaches have historically
struggled &#45; our new algorithm achieved a new state of the art. When evaluated
on Go, chess and shogi, without any knowledge of the game rules, MuZero
matched the superhuman performance of the AlphaZero algorithm that was
supplied with the game rules.

(2019)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1435,-650C1435,-650 1400,-650 1400,-650 1394,-650 1388,-644 1388,-638 1388,-638 1388,-626 1388,-626 1388,-620 1394,-614 1400,-614 1400,-614 1435,-614 1435,-614 1441,-614 1447,-620 1447,-626 1447,-626 1447,-638 1447,-638 1447,-644 1441,-650 1435,-650"/>
<text text-anchor="middle" x="1417.5" y="-628.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MuZero</text>
</a>
</g>
</g>
<!-- Model Based&#45;&gt;MuZero -->
<g id="edge79" class="edge"><title>Model Based&#45;&gt;MuZero</title>
<path fill="none" stroke="black" d="M253.062,-411.012C266.427,-418.945 283.017,-427.659 299,-433 362.489,-454.215 381.711,-445.687 448,-455 507.942,-463.421 523.376,-462.56 583,-473 854.946,-520.617 920.599,-544.626 1190,-605 1218.93,-611.484 1225.69,-615.555 1255,-620 1296.64,-626.314 1344.84,-629.347 1377.67,-630.778"/>
<polygon fill="black" stroke="black" points="1377.63,-634.279 1387.76,-631.188 1377.91,-627.284 1377.63,-634.279"/>
</g>
<!-- Prioritized Sweeping -->
<g id="node68" class="node"><title>Prioritized Sweeping</title>
<g id="a_node68"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#PrioritizedSweeping" xlink:title="Prioritized Sweeping/Queue&#45;Dyna is similar to Dyna, and it improves Dyna by
updating value based on priority rather than randomly. Values are also
associated with state rather than state&#45;action.

(1993)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M571,-590C571,-590 460,-590 460,-590 454,-590 448,-584 448,-578 448,-578 448,-566 448,-566 448,-560 454,-554 460,-554 460,-554 571,-554 571,-554 577,-554 583,-560 583,-566 583,-566 583,-578 583,-578 583,-584 577,-590 571,-590"/>
<text text-anchor="middle" x="515.5" y="-568.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">Prioritized Sweeping</text>
</a>
</g>
</g>
<!-- Dyna&#45;Q&#45;&gt;Prioritized Sweeping -->
<g id="edge80" class="edge"><title>Dyna&#45;Q&#45;&gt;Prioritized Sweeping</title>
<path fill="none" stroke="black" d="M515.5,-518.281C515.5,-526.828 515.5,-535.374 515.5,-543.921"/>
<polygon fill="black" stroke="black" points="512,-543.954 515.5,-553.954 519,-543.954 512,-543.954"/>
</g>
<!-- DMRL -->
<g id="node70" class="node"><title>DMRL</title>
<g id="a_node70"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#DMRL" xlink:title="Deep Meta RL. (from the abstract) In recent years deep reinforcement learning
(RL) systems have attained superhuman performance in a number of challenging
task domains. However, a major limitation of such applications is their demand
for massive amounts of training data. A critical present objective is thus to
develop deep RL methods that can adapt rapidly to new tasks. In the present
work we introduce a novel approach to this challenge, which we refer to as
deep meta&#45;reinforcement learning. Previous work has shown that recurrent
networks can support meta&#45;learning in a fully supervised context. We extend
this approach to the RL setting. What emerges is a system that is trained
using one RL algorithm, but whose recurrent dynamics implement a second, quite
separate RL procedure. This second, learned RL algorithm can differ from the
original one in arbitrary ways. Importantly, because it is learned, it is
configured to exploit structure in the training domain. We unpack these points
in a series of seven proof&#45;of&#45;concept experiments, each of which examines a
key aspect of deep meta&#45;RL. We consider prospects for extending and scaling up
the approach, and also point out some potentially important implications for
neuroscience.

(2016)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M951.5,-78C951.5,-78 921.5,-78 921.5,-78 915.5,-78 909.5,-72 909.5,-66 909.5,-66 909.5,-54 909.5,-54 909.5,-48 915.5,-42 921.5,-42 921.5,-42 951.5,-42 951.5,-42 957.5,-42 963.5,-48 963.5,-54 963.5,-54 963.5,-66 963.5,-66 963.5,-72 957.5,-78 951.5,-78"/>
<text text-anchor="middle" x="936.5" y="-56.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">DMRL</text>
</a>
</g>
</g>
<!-- Meta&#45;RL&#45;&gt;DMRL -->
<g id="edge81" class="edge"><title>Meta&#45;RL&#45;&gt;DMRL</title>
<path fill="none" stroke="black" d="M679.798,-142.908C686.831,-138.635 694.539,-134.352 702,-131 769.012,-100.891 852.363,-78.9977 899.18,-68.003"/>
<polygon fill="black" stroke="black" points="900.198,-71.3599 909.153,-65.6966 898.621,-64.5399 900.198,-71.3599"/>
</g>
<!-- RL^2 -->
<g id="node71" class="node"><title>RL^2</title>
<g id="a_node71"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#RL2" xlink:title="(from the abstract) Deep reinforcement learning (deep RL) has been successful
in learning sophisticated behaviors automatically; however, the learning
process requires a huge number of trials. In contrast, animals can learn new
tasks in just a few trials, benefiting from their prior knowledge about the
world. This paper seeks to bridge this gap. Rather than designing a &quot;fast&quot;
reinforcement learning algorithm, we propose to represent it as a recurrent
neural network (RNN) and learn it from data. In our proposed method, RL2, the
algorithm is encoded in the weights of the RNN, which are learned slowly
through a general&#45;purpose (&quot;slow&quot;) RL algorithm. The RNN receives all
information a typical RL algorithm would receive, including observations,
actions, rewards, and termination flags; and it retains its state across
episodes in a given Markov Decision Process (MDP). The activations of the RNN
store the state of the &quot;fast&quot; RL algorithm on the current (previously unseen)
MDP. We evaluate RL2 experimentally on both small&#45;scale and large&#45;scale
problems. On the small&#45;scale side, we train it to solve randomly generated
multi&#45;arm bandit problems and finite MDPs. After RL2 is trained, its
performance on new MDPs is close to human&#45;designed algorithms with optimality
guarantees. On the large&#45;scale side, we test RL2 on a vision&#45;based navigation
task and show that it scales up to high&#45;dimensional problems.

(2016)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M951.5,-132C951.5,-132 921.5,-132 921.5,-132 915.5,-132 909.5,-126 909.5,-120 909.5,-120 909.5,-108 909.5,-108 909.5,-102 915.5,-96 921.5,-96 921.5,-96 951.5,-96 951.5,-96 957.5,-96 963.5,-102 963.5,-108 963.5,-108 963.5,-120 963.5,-120 963.5,-126 957.5,-132 951.5,-132"/>
<text text-anchor="middle" x="936.5" y="-110.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">RL^2</text>
</a>
</g>
</g>
<!-- Meta&#45;RL&#45;&gt;RL^2 -->
<g id="edge82" class="edge"><title>Meta&#45;RL&#45;&gt;RL^2</title>
<path fill="none" stroke="black" d="M684.169,-150.64C690.048,-148.919 696.179,-147.281 702,-146 770.956,-130.824 853.043,-121.568 899.228,-117.146"/>
<polygon fill="black" stroke="black" points="899.714,-120.616 909.344,-116.2 899.062,-113.646 899.714,-120.616"/>
</g>
<!-- MAML -->
<g id="node72" class="node"><title>MAML</title>
<g id="a_node72"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#MAML" xlink:title="(from the abstract) We propose an algorithm for meta&#45;learning that is model&#45;
agnostic, in the sense that it is compatible with any model trained with
gradient descent and applicable to a variety of different learning problems,
including classification, regression, and reinforcement learning. The goal of
meta&#45;learning is to train a model on a variety of learning tasks, such that it
can solve new learning tasks using only a small number of training samples. In
our approach, the parameters of the model are explicitly trained such that a
small number of gradient steps with a small amount of training data from a new
task will produce good generalization performance on that task. In effect, our
method trains the model to be easy to fine&#45;tune. We demonstrate that this
approach leads to state&#45;of&#45;the&#45;art performance on two few&#45;shot image
classification benchmarks, produces good results on few&#45;shot regression, and
accelerates fine&#45;tuning for policy gradient reinforcement learning with neural
network policies.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-118C1162.5,-118 1132.5,-118 1132.5,-118 1126.5,-118 1120.5,-112 1120.5,-106 1120.5,-106 1120.5,-94 1120.5,-94 1120.5,-88 1126.5,-82 1132.5,-82 1132.5,-82 1162.5,-82 1162.5,-82 1168.5,-82 1174.5,-88 1174.5,-94 1174.5,-94 1174.5,-106 1174.5,-106 1174.5,-112 1168.5,-118 1162.5,-118"/>
<text text-anchor="middle" x="1147.5" y="-96.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">MAML</text>
</a>
</g>
</g>
<!-- Meta&#45;RL&#45;&gt;MAML -->
<g id="edge83" class="edge"><title>Meta&#45;RL&#45;&gt;MAML</title>
<path fill="none" stroke="black" d="M684.097,-158.345C690.038,-157.871 696.21,-157.401 702,-157 827.236,-148.337 859.911,-159.997 984,-141 1028.12,-134.246 1077.71,-120.865 1110.38,-111.224"/>
<polygon fill="black" stroke="black" points="1111.78,-114.457 1120.36,-108.238 1109.77,-107.75 1111.78,-114.457"/>
</g>
<!-- SNAIL -->
<g id="node73" class="node"><title>SNAIL</title>
<g id="a_node73"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#SNAIL" xlink:title="(from the abstract) Deep neural networks excel in regimes with large amounts
of data, but tend to struggle when data is scarce or when they need to adapt
quickly to changes in the task. In response, recent work in meta&#45;learning
proposes training a meta&#45;learner on a distribution of similar tasks, in the
hopes of generalization to novel but related tasks by learning a high&#45;level
strategy that captures the essence of the problem it is asked to solve.
However, many recent meta&#45;learning approaches are extensively hand&#45;designed,
either using architectures specialized to a particular application, or hard&#45;
coding algorithmic components that constrain how the meta&#45;learner solves the
task. We propose a class of simple and generic meta&#45;learner architectures that
use a novel combination of temporal convolutions and soft attention; the
former to aggregate information from past experience and the latter to
pinpoint specific pieces of information. In the most extensive set of meta&#45;
learning experiments to date, we evaluate the resulting Simple Neural
AttentIve Learner (or SNAIL) on several heavily&#45;benchmarked tasks. On all
tasks, in both supervised and reinforcement learning, SNAIL attains state&#45;of&#45;
the&#45;art performance by significant margins.

(2017)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1162.5,-172C1162.5,-172 1132.5,-172 1132.5,-172 1126.5,-172 1120.5,-166 1120.5,-160 1120.5,-160 1120.5,-148 1120.5,-148 1120.5,-142 1126.5,-136 1132.5,-136 1132.5,-136 1162.5,-136 1162.5,-136 1168.5,-136 1174.5,-142 1174.5,-148 1174.5,-148 1174.5,-160 1174.5,-160 1174.5,-166 1168.5,-172 1162.5,-172"/>
<text text-anchor="middle" x="1147.5" y="-150.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">SNAIL</text>
</a>
</g>
</g>
<!-- Meta&#45;RL&#45;&gt;SNAIL -->
<g id="edge84" class="edge"><title>Meta&#45;RL&#45;&gt;SNAIL</title>
<path fill="none" stroke="black" d="M684.053,-160.877C755.29,-160.545 935.957,-159.408 1087,-156 1094.53,-155.83 1102.59,-155.595 1110.28,-155.345"/>
<polygon fill="black" stroke="black" points="1110.56,-158.838 1120.44,-155.001 1110.32,-151.842 1110.56,-158.838"/>
</g>
<!-- ProMP -->
<g id="node74" class="node"><title>ProMP</title>
<g id="a_node74"><a xlink:href="https://github.com/bennylp/RL-Taxonomy#ProMP" xlink:title="ProMP: Proximal Meta&#45;Policy Search (from the abstract) Credit assignment in
Meta&#45;reinforcement learning (Meta&#45;RL) is still poorly understood. Existing
methods either neglect credit assignment to pre&#45;adaptation behavior or
implement it naively. This leads to poor sample&#45;efficiency during meta&#45;
training as well as ineffective task identification strategies. This paper
provides a theoretical analysis of credit assignment in gradient&#45;based Meta&#45;
RL. Building on the gained insights we develop a novel meta&#45;learning algorithm
that overcomes both the issue of poor credit assignment and previous
difficulties in estimating meta&#45;policy gradients. By controlling the
statistical distance of both pre&#45;adaptation and adapted policies during meta&#45;
policy search, the proposed algorithm endows efficient and stable meta&#45;
learning. Our approach leads to superior pre&#45;adaptation policy behavior and
consistently outperforms previous Meta&#45;RL algorithms in sample&#45;efficiency,
wall&#45;clock time, and asymptotic performance.

(2018)">
<path fill="#dae8fc" stroke="black" stroke-width="2" d="M1318,-192C1318,-192 1287,-192 1287,-192 1281,-192 1275,-186 1275,-180 1275,-180 1275,-168 1275,-168 1275,-162 1281,-156 1287,-156 1287,-156 1318,-156 1318,-156 1324,-156 1330,-162 1330,-168 1330,-168 1330,-180 1330,-180 1330,-186 1324,-192 1318,-192"/>
<text text-anchor="middle" x="1302.5" y="-170.9" font-family="Helvetica,sans-Serif" font-weight="bold" font-size="12.00">ProMP</text>
</a>
</g>
</g>
<!-- Meta&#45;RL&#45;&gt;ProMP -->
<g id="edge85" class="edge"><title>Meta&#45;RL&#45;&gt;ProMP</title>
<path fill="none" stroke="black" d="M684.141,-170.246C716.622,-179.06 769.168,-191 815.5,-191 815.5,-191 815.5,-191 1148.5,-191 1188.48,-191 1233.98,-185.195 1264.89,-180.37"/>
<polygon fill="black" stroke="black" points="1265.47,-183.822 1274.79,-178.781 1264.36,-176.91 1265.47,-183.822"/>
</g>
</g>
</svg>