diff --git a/CHANGELOG.md b/CHANGELOG.md
index a46f48983c..e0c5328a63 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - [Pico] Improve performance ([#2724](https://github.com/cucumber/cucumber-jvm/issues/2724) Julien Kronegg)
 - [JUnit 4] Fix swallowed exception ([#2714](https://github.com/cucumber/cucumber-jvm/issues/2714) M.P. Korstanje)
+- [Guice] Fix NPE in Guice when configured incorrectly ([#2716](https://github.com/cucumber/cucumber-jvm/issues/2716) M.P. Korstanje)
 
 ## [7.11.2] - 2023-03-23
 ### Fixed
diff --git a/cucumber-guice/src/main/java/io/cucumber/guice/GuiceFactory.java b/cucumber-guice/src/main/java/io/cucumber/guice/GuiceFactory.java
index 6dcc42cbe0..eb7856cd96 100644
--- a/cucumber-guice/src/main/java/io/cucumber/guice/GuiceFactory.java
+++ b/cucumber-guice/src/main/java/io/cucumber/guice/GuiceFactory.java
@@ -24,10 +24,11 @@ public final class GuiceFactory implements ObjectFactory {
 
     private final Collection<Class<?>> stepClasses = new HashSet<>();
     private final Class<?> injectorSourceFromProperty;
-    private Class<?> withInjectorSource = null;
+    private Class<?> withInjectorSource;
+    private ScenarioScope scenarioScope;
 
     public GuiceFactory() {
-        injectorSourceFromProperty = loadInjectorSourceFromProperties(CucumberProperties.create());
+        this.injectorSourceFromProperty = loadInjectorSourceFromProperties(CucumberProperties.create());
     }
 
     @Override
@@ -72,11 +73,15 @@ public void start() {
             injector = new InjectorSourceFactory(withInjectorSource).create()
                     .getInjector();
         }
-        injector.getInstance(ScenarioScope.class).enterScope();
+        scenarioScope = injector.getInstance(ScenarioScope.class);
+        scenarioScope.enterScope();
     }
 
     public void stop() {
-        injector.getInstance(ScenarioScope.class).exitScope();
+        if (scenarioScope != null) {
+            scenarioScope.exitScope();
+            scenarioScope = null;
+        }
     }
 
     public <T> T getInstance(Class<T> clazz) {
diff --git a/cucumber-guice/src/test/java/io/cucumber/guice/GuiceFactoryTest.java b/cucumber-guice/src/test/java/io/cucumber/guice/GuiceFactoryTest.java
index e8d515fcd8..53c9334a67 100644
--- a/cucumber-guice/src/test/java/io/cucumber/guice/GuiceFactoryTest.java
+++ b/cucumber-guice/src/test/java/io/cucumber/guice/GuiceFactoryTest.java
@@ -26,9 +26,7 @@
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.notNullValue;
-import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.text.IsEqualCompressingWhiteSpace.equalToCompressingWhiteSpace;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
@@ -52,9 +50,8 @@ protected void configure() {
     void tearDown() {
         // If factory is left in start state it can cause cascading failures due
         // to scope being left open
-        try {
+        if (factory != null) {
             factory.stop();
-        } catch (Exception ignored) {
         }
     }
 
@@ -80,8 +77,7 @@ void factoryCanBeInstantiatedWithArgConstructor() {
     void factoryStartFailsIfScenarioScopeIsNotBound() {
         initFactory(Guice.createInjector());
 
-        Executable testMethod = () -> factory.start();
-        ConfigurationException actualThrown = assertThrows(ConfigurationException.class, testMethod);
+        ConfigurationException actualThrown = assertThrows(ConfigurationException.class, () -> factory.start());
         assertThat("Unexpected exception message", actualThrown.getMessage(),
             containsString("1) [Guice/MissingImplementation]: No implementation for ScenarioScope was bound."));
     }