diff --git a/openwakeword/model.py b/openwakeword/model.py
index 72524ac..b530696 100755
--- a/openwakeword/model.py
+++ b/openwakeword/model.py
@@ -39,7 +39,7 @@ class Model():
             class_mapping_dicts: List[dict] = [],
             enable_speex_noise_suppression: bool = False,
             vad_threshold: float = 0,
-            custom_verifier_models: Union[bool, dict] = False,
+            custom_verifier_models: dict = {},
             custom_verifier_threshold: float = 0.1,
             **kwargs
             ):
@@ -112,6 +112,14 @@ class Model():
                 if custom_verifier_models.get(mdl_name, False):
                     self.custom_verifier_models[mdl_name] = pickle.load(open(custom_verifier_models[mdl_name], 'rb'))
 
+            if len(self.custom_verifier_models.keys()) < len(custom_verifier_models.keys()):
+                raise ValueError(
+                    "Custom verifier models were provided, but some were not matched with a base model!"
+                    " Make sure that the keys provided in the `custom_verifier_models` dictionary argument"
+                    " exactly match that of the `.models` attribute of an instantiated openWakeWord Model object"
+                    " that has the same base models but doesn't have custom verifier models."
+                )
+
         # Create buffer to store frame predictions
         self.prediction_buffer: DefaultDict[str, deque] = defaultdict(partial(deque, maxlen=30))
 
@@ -208,10 +216,11 @@ class Model():
                 for cls in predictions.keys():
                     if predictions[cls] >= self.custom_verifier_threshold:
                         parent_model = self.get_parent_model_from_label(cls)
-                        verifier_prediction = self.custom_verifier_models[parent_model].predict_proba(
-                            self.preprocessor.get_features(self.model_inputs[mdl])
-                        )[0][-1]
-                        predictions[cls] = verifier_prediction
+                        if self.custom_verifier_models.get(parent_model, False):
+                            verifier_prediction = self.custom_verifier_models[parent_model].predict_proba(
+                                self.preprocessor.get_features(self.model_inputs[mdl])
+                            )[0][-1]
+                            predictions[cls] = verifier_prediction
 
             # Update prediction buffer, and zero predictions for first 5 frames during model initialization
             for cls in predictions.keys():
diff --git a/pyproject.toml b/pyproject.toml
index 4b26990..f922c05 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -12,7 +12,7 @@ testpaths = [
 
 [project]
 name = "openwakeword"
-version = "0.3.0"
+version = "0.3.1"
 authors = [
   { name="David Scripka", email="david.scripka@gmail.com" },
 ]
diff --git a/setup.py b/setup.py
index 1d0a85b..fc835ee 100644
--- a/setup.py
+++ b/setup.py
@@ -26,7 +26,7 @@ def build_additional_requires():
 
 setuptools.setup(
     name="openwakeword",
-    version="0.3.0",
+    version="0.3.1",
     install_requires=['onnxruntime>=1.10.0,<2', 'tqdm>=4.0,<5.0', 'scipy>=1.3,<2', 'scikit-learn>=1,<2'],
     extras_require={
         'test': [
diff --git a/tests/test_custom_verifier_model.py b/tests/test_custom_verifier_model.py
index 0e043f5..6ca6d1d 100644
--- a/tests/test_custom_verifier_model.py
+++ b/tests/test_custom_verifier_model.py
@@ -72,7 +72,15 @@ class TestModels:
                 model_name=os.path.join("openwakeword", "resources", "models", "hey_mycroft_v0.1.onnx")
             )
 
-            # Load model with verifier model
+            with pytest.raises(ValueError):
+                # Load model with verifier model incorrectly to catch ValueError
+                owwModel = openwakeword.Model(
+                    wakeword_model_paths=[os.path.join("openwakeword", "resources", "models", "hey_mycroft_v0.1.onnx")],
+                    custom_verifier_models={"bad_key": os.path.join(tmp_dir, "verifier_model.pkl")},
+                    custom_verifier_threshold=0.3,
+                )
+
+            # Load model with verifier model incorrectly to catch ValueError
             owwModel = openwakeword.Model(
                 wakeword_model_paths=[os.path.join("openwakeword", "resources", "models", "hey_mycroft_v0.1.onnx")],
                 custom_verifier_models={"hey_mycroft_v0.1": os.path.join(tmp_dir, "verifier_model.pkl")},