diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 8364be9b8..96a5bd720 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -78,7 +78,7 @@ jobs:
 
         - script: 'python setup.py build_sphinx -b doctest'
           displayName: 'Run doctests'
-      package: '-e .[automl]'
+      package: '-e .[all]'
 
 - job: 'Notebooks'
   dependsOn: 'EvalChanges'
@@ -106,6 +106,7 @@ jobs:
             testResultsFiles: '**/test-results.xml'
             testRunTitle: 'Notebooks'
           condition: succeededOrFailed()
+      package: '-e .[tf,interpret]'
 
 # - job: 'AutoML'
 #   dependsOn: 'EvalChanges'
@@ -216,3 +217,4 @@ jobs:
           inputs:
             codeCoverageTool: Cobertura
             summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/coverage.xml'
+      package: '-e .[tf,interpret]'
diff --git a/econml/cate_interpreter/_interpreters.py b/econml/cate_interpreter/_interpreters.py
index f3147f373..4548e9e09 100644
--- a/econml/cate_interpreter/_interpreters.py
+++ b/econml/cate_interpreter/_interpreters.py
@@ -6,8 +6,8 @@
 from io import StringIO
 from sklearn.tree import DecisionTreeRegressor, DecisionTreeClassifier
 from sklearn.utils.validation import check_is_fitted
-import graphviz
 from ._tree_exporter import _CateTreeDOTExporter, _CateTreeMPLExporter, _PolicyTreeDOTExporter, _PolicyTreeMPLExporter
+import graphviz
 
 
 class _SingleTreeInterpreter(metaclass=abc.ABCMeta):
diff --git a/econml/cate_interpreter/_tree_exporter.py b/econml/cate_interpreter/_tree_exporter.py
index b4232493e..cf67fbd1a 100644
--- a/econml/cate_interpreter/_tree_exporter.py
+++ b/econml/cate_interpreter/_tree_exporter.py
@@ -3,9 +3,18 @@
 
 import numpy as np
 import re
-import matplotlib
-matplotlib.use('Agg')
-import matplotlib.pyplot as plt
+try:
+    import matplotlib
+    matplotlib.use('Agg')
+    import matplotlib.pyplot as plt
+except ImportError as exn:
+    from ..utilities import MissingModule
+
+    # make any access to matplotlib or plt throw an exception
+    matplotlib = plt = MissingModule("matplotlib is no longer a dependency of the main econml package; "
+                                     "install econml[plt] or econml[all] to require them, or install matplotlib "
+                                     "separately, to use the tree interpreters", exn)
+
 
 # HACK: We're relying on some of sklearn's non-public classes which are not completely stable.
 #       However, the alternative is reimplementing a bunch of intricate stuff by hand
diff --git a/econml/iv/nnet/_deepiv.py b/econml/iv/nnet/_deepiv.py
index 6f5a8369b..18123b0d7 100644
--- a/econml/iv/nnet/_deepiv.py
+++ b/econml/iv/nnet/_deepiv.py
@@ -4,13 +4,17 @@
 """Deep IV estimator and related components."""
 
 import numpy as np
-import keras
 from ..._cate_estimator import BaseCateEstimator
-from ...utilities import deprecated
-from keras import backend as K
-import keras.layers as L
-from keras.models import Model
-from econml.utilities import check_input_arrays, _deprecate_positional
+from ...utilities import check_input_arrays, _deprecate_positional, deprecated, MissingModule
+try:
+    import keras
+    from keras import backend as K
+    import keras.layers as L
+    from keras.models import Model
+except ImportError as exn:
+    keras = K = L = Model = MissingModule("keras and tensorflow are no longer dependencies of the main econml "
+                                          "package; install econml[tf] or econml[all] to require them, or install "
+                                          "them separately, to use DeepIV", exn)
 
 # TODO: make sure to use random seeds wherever necessary
 # TODO: make sure that the public API consistently uses "T" instead of "P" for the treatment
diff --git a/econml/utilities.py b/econml/utilities.py
index dbcd02e44..212f74b0d 100644
--- a/econml/utilities.py
+++ b/econml/utilities.py
@@ -1231,6 +1231,31 @@ def m(*args, **kwargs):
     return decorator
 
 
+class MissingModule:
+    """
+    Placeholder to stand in for a module that couldn't be imported, delaying ImportErrors until use.
+
+    Parameters
+    ----------
+    msg:string
+        The message to display when an attempt to access a module memeber is made
+    exn:ImportError
+        The original ImportError to pass as the source of the exception
+    """
+
+    def __init__(self, msg, exn):
+        self.msg = msg
+        self.exn = exn
+
+    # Any access should throw
+    def __getattr__(self, _):
+        raise ImportError(self.msg) from self.exn
+
+    # As a convenience, also throw on calls to allow MissingModule to be used in lieu of specific imports
+    def __call__(self, *args, **kwargs):
+        raise ImportError(self.msg) from self.exn
+
+
 def transpose_dictionary(d):
     """
     Transpose a dictionary of dictionaries, bringing the keys from the second level
diff --git a/setup.cfg b/setup.cfg
index c7bfd48d4..77f92afdb 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -43,14 +43,11 @@ install_requires =
     numpy
     scipy > 1.4.0
     scikit-learn >= 0.24
-    keras < 2.4
     sparse
-    tensorflow > 1.10, < 2.3
     joblib >= 0.13.0
     numba != 0.42.1
     statsmodels >= 0.9
     graphviz
-    matplotlib
     pandas
     shap ~= 0.38.1
     dowhy
@@ -71,6 +68,16 @@ automl =
     ; Disabled due to incompatibility with scikit-learn
     ; azureml-sdk[explain,automl] == 1.0.83
     azure-cli
+tf =
+    keras < 2.4
+    tensorflow > 1.10, < 2.3
+plt =
+    matplotlib
+all =
+    azure-cli
+    keras < 2.4
+    tensorflow > 1.10, < 2.3
+    matplotlib
     
 ; TODO: exclude tests?
 [options.packages.find]