Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix coverage issue in BC tests #830

Merged
merged 5 commits into from
Dec 15, 2023
Merged

Fix coverage issue in BC tests #830

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
71afc08
Remove custom logger from hypothesis strategies to make the number of…
ernestum Dec 11, 2023
6e56748
Test BC with different batch_size/minibatch_size combinations.
ernestum Dec 11, 2023
cfd0781
Add some no cover pragmas to unimportant edge-cases of BC.
ernestum Dec 11, 2023
293015c
Fix formatting
ernestum Dec 11, 2023
4a34039
Add better A note explaining why we suppress certain hypothesis healt…
ernestum Dec 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/imitation/algorithms/bc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ def __init__(
self._demo_data_loader: Optional[Iterable[types.TransitionMapping]] = None
self.batch_size = batch_size
self.minibatch_size = minibatch_size or batch_size
if self.batch_size % self.minibatch_size != 0:
if self.batch_size % self.minibatch_size != 0: # pragma: no cover
raise ValueError("Batch size must be a multiple of minibatch size.")
super().__init__(
demonstrations=demonstrations,
Expand Down Expand Up @@ -358,7 +358,7 @@ def __init__(
assert self.policy.action_space == self.action_space

if optimizer_kwargs:
if "weight_decay" in optimizer_kwargs:
if "weight_decay" in optimizer_kwargs: # pragma: no cover
raise ValueError("Use the parameter l2_weight instead of weight_decay.")
optimizer_kwargs = optimizer_kwargs or {}
self.optimizer = optimizer_cls(
Expand Down
23 changes: 13 additions & 10 deletions tests/algorithms/test_bc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ def make_bc_train_args(
rng=rngs,
)
batch_sizes = st.integers(min_value=1, max_value=50)
loggers = st.sampled_from([None, logger.configure()])
expert_data_types = st.sampled_from(
["data_loader", "ducktyped_data_loader", "transitions"],
)
Expand All @@ -95,17 +94,17 @@ def make_bc_train_args(
log_rollouts_venv=st.one_of(rollout_envs, st.just(None)),
)
bc_args = st.builds(
lambda env, batch_size, custom_logger, rng: dict(
lambda env, minibatch_size, rng, minibatch_fraction: dict(
observation_space=env.observation_space,
action_space=env.action_space,
batch_size=batch_size,
custom_logger=custom_logger,
batch_size=minibatch_size * minibatch_fraction,
minibatch_size=minibatch_size,
rng=rng,
),
env=envs,
batch_size=batch_sizes,
custom_logger=loggers,
minibatch_size=batch_sizes,
rng=rngs,
minibatch_fraction=st.integers(1, 10),
)


Expand Down Expand Up @@ -136,7 +135,7 @@ def test_smoke_bc_creation(
**bc_args,
demonstrations=make_expert_transition_loader(
cache_dir=cache.mkdir("experts"),
batch_size=bc_args["batch_size"],
batch_size=bc_args["minibatch_size"],
expert_data_type=expert_data_type,
env_name=env_name,
rng=rng,
Expand All @@ -152,7 +151,12 @@ def test_smoke_bc_creation(
expert_data_type=expert_data_types,
rng=rngs,
)
@hypothesis.settings(deadline=20000, max_examples=15)
@hypothesis.settings(
deadline=20000,
max_examples=15,
# TODO: one day consider removing this. For now we are good.
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe this is obvious for someone who's more familiar with hypothesis or this codebase but what does this check do? would it be helpful to clarify when it would be possible to remove this check or what's going that causes this check to fail?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks that is a good point! I added a better description. Does it make sense now @tomtseng ?

suppress_health_check=[hypothesis.HealthCheck.data_too_large],
)
def test_smoke_bc_training(
env_name: str,
bc_args: dict,
Expand All @@ -168,7 +172,7 @@ def test_smoke_bc_training(
**bc_args,
demonstrations=make_expert_transition_loader(
cache_dir=cache.mkdir("experts"),
batch_size=bc_args["batch_size"],
batch_size=bc_args["minibatch_size"],
expert_data_type=expert_data_type,
env_name=env_name,
rng=rng,
Expand Down Expand Up @@ -246,7 +250,6 @@ def make_trainer(**kwargs: Any) -> bc.BC:
action_space=cartpole_venv.action_space,
batch_size=batch_size,
demonstrations=demonstrations,
custom_logger=None,
rng=rng,
**kwargs,
)
Expand Down