[RLlib] Add PPO StatelessCarePole learning tests (+LSTM) to CI.

rllib/BUILD

@@ -190,7 +190,7 @@ py_test(
     tags = ["team:rllib", "exclusive", "learning_tests", "learning_tests_cartpole", "learning_tests_discrete", "torch_only"],
     size = "large",
     srcs = ["tuned_examples/appo/multi_agent_cartpole_appo.py"],
-    args = ["--as-test", "--enable-new-api-stack", "--num-agents=2", "--num-gpus=1"]
+    args = ["--as-test", "--enable-new-api-stack", "--num-agents=2", "--num-gpus=1", "--num-cpus=6"]
 )
 py_test(
     name = "learning_tests_multi_agent_cartpole_appo_gpu",

rllib/tuned_examples/appo/multi_agent_cartpole_appo.py

@@ -9,7 +9,7 @@
 from ray.tune.registry import register_env
 
 parser = add_rllib_example_script_args()
-parser.set_defaults(num_agents=2)
+parser.set_defaults(num_agents=2, num_env_runners=5)
 # Use `parser` to add your own custom command line options to this script
 # and (if needed) use their values toset up `config` below.
 args = parser.parse_args()
@@ -26,8 +26,9 @@
     )
     .environment("env", env_config={"num_agents": args.num_agents})
     .training(
-        vf_loss_coeff=0.005,
-        entropy_coeff=0.0,
+        vf_loss_coeff=0.05,
+        lr=0.001,
+        entropy_coeff=0.1,
     )
     .rl_module(
         model_config_dict={

rllib/tuned_examples/impala/multi_agent_cartpole_impala.py

@@ -9,7 +9,7 @@
 from ray.tune.registry import register_env
 
 parser = add_rllib_example_script_args()
-parser.set_defaults(num_agents=2, num_env_runners=4)
+parser.set_defaults(num_agents=2, num_env_runners=5)
 # Use `parser` to add your own custom command line options to this script
 # and (if needed) use their values toset up `config` below.
 args = parser.parse_args()

rllib/utils/test_utils.py

@@ -1474,7 +1474,6 @@ def run_rllib_example_script_experiment(
         # New stack.
         if config.enable_rl_module_and_learner:
             # Define compute resources used.
-            config.resources(num_gpus=0)
             config.learners(
                 num_learners=args.num_gpus,
                 num_gpus_per_learner=(