Skip to content

Commit

Permalink
haul over analyse faces
Browse files Browse the repository at this point in the history
  • Loading branch information
iulusoy committed Jun 12, 2024
1 parent 55d9ee9 commit 39a617a
Show file tree
Hide file tree
Showing 7 changed files with 84 additions and 127 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ Use pre-processed image files such as social media posts with comments and proce
1. Question answering
1. Performing person and face recognition in images
1. Face mask detection
1. Age, gender and race detection
1. Probabilistic detection of age, gender and race detection
1. Emotion recognition
1. Color analysis
1. Analyse hue and percentage of color on image
Expand Down
34 changes: 0 additions & 34 deletions ammico/display.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,6 @@ def __init__(self, mydict: dict) -> None:
State("setting_Text_revision_numbers", "value"),
State("setting_Emotion_emotion_threshold", "value"),
State("setting_Emotion_race_threshold", "value"),
State("setting_Emotion_gender_threshold", "value"),
State("setting_Emotion_age_threshold", "value"),
State("setting_Emotion_env_var", "value"),
State("setting_Color_delta_e_method", "value"),
State("setting_Summary_analysis_type", "value"),
Expand Down Expand Up @@ -256,34 +254,6 @@ def _create_setting_layout(self):
],
align="start",
),
dbc.Col(
[
html.P("Gender threshold"),
dcc.Input(
type="number",
value=50,
max=100,
min=0,
id="setting_Emotion_gender_threshold",
style={"width": "100%"},
),
],
align="start",
),
dbc.Col(
[
html.P("Age threshold"),
dcc.Input(
type="number",
value=50,
max=100,
min=0,
id="setting_Emotion_age_threshold",
style={"width": "100%"},
),
],
align="start",
),
dbc.Col(
[
html.P(
Expand Down Expand Up @@ -493,8 +463,6 @@ def _right_output_analysis(
settings_text_revision_numbers: str,
setting_emotion_emotion_threshold: int,
setting_emotion_race_threshold: int,
setting_emotion_gender_threshold: int,
setting_emotion_age_threshold: int,
setting_emotion_env_var: str,
setting_color_delta_e_method: str,
setting_summary_analysis_type: str,
Expand Down Expand Up @@ -550,8 +518,6 @@ def _right_output_analysis(
image_copy,
emotion_threshold=setting_emotion_emotion_threshold,
race_threshold=setting_emotion_race_threshold,
gender_threshold=setting_emotion_gender_threshold,
age_threshold=setting_emotion_age_threshold,
accept_disclosure=(
setting_emotion_env_var
if setting_emotion_env_var
Expand Down
105 changes: 46 additions & 59 deletions ammico/faces.py
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,6 @@ def __init__(
subdict: dict,
emotion_threshold: float = 50.0,
race_threshold: float = 50.0,
gender_threshold: float = 50.0,
age_threshold: float = 50.0,
accept_disclosure: str = "DISCLOSURE_AMMICO",
) -> None:
"""
Expand All @@ -160,8 +158,6 @@ def __init__(
subdict (dict): The dictionary to store the analysis results.
emotion_threshold (float): The threshold for detecting emotions (default: 50.0).
race_threshold (float): The threshold for detecting race (default: 50.0).
gender_threshold (float): The threshold for detecting gender (default: 50.0).
age_threshold (float): The threshold for detecting age (default: 50.0).
accept_disclosure (str): The name of the disclosure variable, that is
set upon accepting the disclosure (default: "DISCLOSURE_AMMICO").
"""
Expand All @@ -172,14 +168,8 @@ def __init__(
raise ValueError("Emotion threshold must be between 0 and 100.")
if race_threshold < 0 or race_threshold > 100:
raise ValueError("Race threshold must be between 0 and 100.")
if gender_threshold < 0 or gender_threshold > 100:
raise ValueError("Gender threshold must be between 0 and 100.")
if age_threshold < 0 or age_threshold > 100:
raise ValueError("Age threshold must be between 0 and 100.")
self.emotion_threshold = emotion_threshold
self.race_threshold = race_threshold
self.gender_threshold = gender_threshold
self.age_threshold = age_threshold
self.emotion_categories = {
"angry": "Negative",
"disgust": "Negative",
Expand Down Expand Up @@ -232,30 +222,29 @@ def _define_actions(self, fresult: dict) -> list:
"restricted_access_with_mask": [],
}
if fresult["wears_mask"] and self.accepted:
actions = conditional_actions["all_with_mask"]
self.actions = conditional_actions["all_with_mask"]
elif fresult["wears_mask"] and not self.accepted:
actions = conditional_actions["restricted_access_with_mask"]
self.actions = conditional_actions["restricted_access_with_mask"]
elif not fresult["wears_mask"] and self.accepted:
actions = conditional_actions["all"]
self.actions = conditional_actions["all"]
elif not fresult["wears_mask"] and not self.accepted:
actions = conditional_actions["restricted_access"]
self.actions = conditional_actions["restricted_access"]
else:
raise ValueError(
"Invalid mask detection {} and disclosure \
acceptance {} result.".format(
fresult["wears_mask"], self.accepted
)
)
return actions

def _ensure_deepface_models(self, actions: list):
def _ensure_deepface_models(self):
# Ensure that all data has been fetched by pooch
deepface_face_expression_model.get()
if "race" in actions:
if "race" in self.actions:
deepface_race_model.get()
if "age" in actions:
if "age" in self.actions:
deepface_age_model.get()
if "gender" in actions:
if "gender" in self.actions:
deepface_gender_model.get()

def analyze_single_face(self, face: np.ndarray) -> dict:
Expand All @@ -271,13 +260,13 @@ def analyze_single_face(self, face: np.ndarray) -> dict:
fresult = {}
# Determine whether the face wears a mask
fresult["wears_mask"] = self.wears_mask(face)
actions = self._define_actions(fresult)
self._ensure_deepface_models(actions)
self._define_actions(fresult)
self._ensure_deepface_models()
# Run the full DeepFace analysis
fresult.update(
DeepFace.analyze(
img_path=face,
actions=actions,
actions=self.actions,
prog_bar=False,
detector_backend="skip",
)
Expand Down Expand Up @@ -325,49 +314,47 @@ def clean_subdict(self, result: dict) -> dict:
"""
# Each person subdict converted into list for keys
self.subdict["wears_mask"] = []
self.subdict["age"] = []
self.subdict["gender"] = []
self.subdict["race"] = []
self.subdict["emotion"] = []
self.subdict["emotion (category)"] = []
for key in self.actions:
self.subdict[key] = []
# now iterate over the number of faces
# and check thresholds
# the results for each person are returned as a nested dict
# race and emotion are given as dict with confidence values
# gender and age are given as one value with no confidence
# being passed
for i in range(result["number_faces"]):
person = "person{}".format(i + 1)
self.subdict["wears_mask"].append(
"Yes" if result[person]["wears_mask"] else "No"
)
self.subdict["age"].append(result[person]["age"])
# Gender is now reported as a list of dictionaries.
# Each dict represents one face.
# Each dict contains probability for Woman and Man.
# We take only the higher probability result for each dict.
self.subdict["gender"].append(result[person]["gender"])
# Race and emotion are only detected if a person does not wear a mask
if result[person]["wears_mask"]:
self.subdict["race"].append(None)
self.subdict["emotion"].append(None)
self.subdict["emotion (category)"].append(None)
elif not result[person]["wears_mask"]:
# Check whether the race threshold was exceeded
if (
result[person]["race"][result[person]["dominant_race"]]
> self.race_threshold
):
self.subdict["race"].append(result[person]["dominant_race"])
else:
self.subdict["race"].append(None)

# Check whether the emotion threshold was exceeded
if (
result[person]["emotion"][result[person]["dominant_emotion"]]
> self.emotion_threshold
):
self.subdict["emotion"].append(result[person]["dominant_emotion"])
wears_mask = result[person]["wears_mask"]
self.subdict["wears_mask"].append("Yes" if wears_mask else "No")
for key in self.actions:
if key == "emotion":
classified_emotion = result[person]["dominant_emotion"]
confidence_value = result[person][key][classified_emotion]
outcome = (
classified_emotion
if confidence_value > self.emotion_threshold and not wears_mask
else None
)
# also set the emotion category
self.emotion_categories[outcome]
self.subdict["emotion (category)"].append(
self.emotion_categories[result[person]["dominant_emotion"]]
self.emotion_categories[outcome] if outcome else None
)
elif key == "race":
classified_race = result[person]["dominant_race"]
confidence_value = result[person][key][classified_race]
outcome = (
classified_race
if confidence_value > self.race_threshold and not wears_mask
else None
)
else:
self.subdict["emotion"].append(None)
self.subdict["emotion (category)"].append(None)
elif key == "age":
outcome = result[person]["age"] if not wears_mask else None
elif key == "gender":
outcome = result[person]["gender"] if not wears_mask else None
self.subdict[key].append(outcome)
return self.subdict

def wears_mask(self, face: np.ndarray) -> bool:
Expand Down
26 changes: 23 additions & 3 deletions ammico/notebooks/DemoNotebook_ammico.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -261,13 +261,32 @@
" image_df.to_csv(dump_file)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"image_df.head()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"`TextDetector`:"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"os.environ.pop(\"DISCLOSURE_AMMICO\")\n",
"os.environ.get(\"DISCLOSURE_AMMICO\")"
]
},
{
"cell_type": "code",
"execution_count": null,
Expand Down Expand Up @@ -876,11 +895,13 @@
"\n",
"From the seven facial expressions, an overall dominating emotion category is identified: negative, positive, or neutral emotion. These are defined with the facial expressions angry, disgust, fear and sad for the negative category, happy for the positive category, and surprise and neutral for the neutral category.\n",
"\n",
"A similar threshold as for the emotion recognition is set for the race/ethnicity, gender and age detection, `race_threshold`, `gender_threshold`, `age_threshold`, with the default set to 50% so that a confidence for the race above 0.5 only will return a value in the analysis. \n",
"A similar threshold as for the emotion recognition is set for the race/ethnicity detection, `race_threshold`, with the default set to 50% so that a confidence for the race above 0.5 only will return a value in the analysis. \n",
"\n",
"For age and gender, unfortunately no confidence value is accessible so that no threshold values can be set for this type of analysis.\n",
"\n",
"You may also pass the name of the environment variable that determines if you accept or reject the ethical disclosure statement. By default, the variable is named `DISCLOSURE_AMMICO`.\n",
"\n",
"Summarizing, the face detection is carried out using the following method call and keywords, where `emotion_threshold`, `race_threshold`, `gender_threshold`, `age_threshold` are optional:"
"Summarizing, the face detection is carried out using the following method call and keywords, where `emotion_threshold`, `race_threshold`, `accept_disclosure` are optional:"
]
},
{
Expand All @@ -891,7 +912,6 @@
"source": [
"for key in image_dict.keys():\n",
" image_dict[key] = ammico.EmotionDetector(image_dict[key], emotion_threshold=50, race_threshold=50,\n",
" gender_threshold=50, age_threshold=50, \n",
" accept_disclosure=\"DISCLOSURE_AMMICO\").analyse_image()"
]
},
Expand Down
4 changes: 0 additions & 4 deletions ammico/test/test_display.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,6 @@ def test_right_output_analysis_summary(get_AE, get_options, monkeypatch):
None,
50,
50,
50,
50,
"OTHER_VAR",
"CIE 1976",
"summary_and_questions",
Expand All @@ -76,8 +74,6 @@ def test_right_output_analysis_emotions(get_AE, get_options, monkeypatch):
None,
50,
50,
50,
50,
"OTHER_VAR",
"CIE 1976",
"summary_and_questions",
Expand Down
33 changes: 10 additions & 23 deletions ammico/test/test_faces.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@ def test_init_EmotionDetector(monkeypatch):
assert ed.subdict["emotion"] == [None]
assert ed.subdict["age"] == [None]
assert ed.emotion_threshold == 50
assert ed.age_threshold == 50
assert ed.gender_threshold == 50
assert ed.race_threshold == 50
assert ed.emotion_categories["angry"] == "Negative"
assert ed.emotion_categories["happy"] == "Positive"
Expand All @@ -27,14 +25,10 @@ def test_init_EmotionDetector(monkeypatch):
{},
emotion_threshold=80,
race_threshold=30,
gender_threshold=70,
age_threshold=90,
accept_disclosure="OTHER_VAR",
)
assert ed.emotion_threshold == 80
assert ed.race_threshold == 30
assert ed.gender_threshold == 70
assert ed.age_threshold == 90
monkeypatch.delenv("OTHER_VAR", raising=False)
# do not accept disclosure
monkeypatch.setattr("builtins.input", lambda _: "no")
Expand All @@ -52,14 +46,6 @@ def test_init_EmotionDetector(monkeypatch):
fc.EmotionDetector({}, race_threshold=150)
with pytest.raises(ValueError):
fc.EmotionDetector({}, race_threshold=-50)
with pytest.raises(ValueError):
fc.EmotionDetector({}, gender_threshold=150)
with pytest.raises(ValueError):
fc.EmotionDetector({}, gender_threshold=-50)
with pytest.raises(ValueError):
fc.EmotionDetector({}, age_threshold=150)
with pytest.raises(ValueError):
fc.EmotionDetector({}, age_threshold=-50)
# test pre-set variables: disclosure
monkeypatch.delattr("builtins.input", raising=False)
monkeypatch.setenv("OTHER_VAR", "something")
Expand All @@ -76,22 +62,23 @@ def test_init_EmotionDetector(monkeypatch):
def test_define_actions(monkeypatch):
monkeypatch.setenv("OTHER_VAR", "True")
ed = fc.EmotionDetector({}, accept_disclosure="OTHER_VAR")
actions = ed._define_actions({"wears_mask": True})
assert actions == ["age", "gender"]
actions = ed._define_actions({"wears_mask": False})
assert actions == ["age", "gender", "race", "emotion"]
ed._define_actions({"wears_mask": True})
assert ed.actions == ["age", "gender"]
ed._define_actions({"wears_mask": False})
assert ed.actions == ["age", "gender", "race", "emotion"]
monkeypatch.setenv("OTHER_VAR", "False")
ed = fc.EmotionDetector({}, accept_disclosure="OTHER_VAR")
actions = ed._define_actions({"wears_mask": True})
assert actions == []
actions = ed._define_actions({"wears_mask": False})
assert actions == ["emotion"]
ed._define_actions({"wears_mask": True})
assert ed.actions == []
ed._define_actions({"wears_mask": False})
assert ed.actions == ["emotion"]


def test_ensure_deepface_models(monkeypatch):
monkeypatch.setenv("OTHER_VAR", "True")
ed = fc.EmotionDetector({}, accept_disclosure="OTHER_VAR")
ed._ensure_deepface_models(["age", "gender", "race", "emotion"])
ed.actions = ["age", "gender", "race", "emotion"]
ed._ensure_deepface_models()


def test_analyse_faces(get_path, monkeypatch):
Expand Down
Loading

0 comments on commit 39a617a

Please sign in to comment.