Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add image.detect() for detecting multiple types. #2770

Merged
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Add image.detect() for detecting multiple types.
  • Loading branch information
daspecster committed Nov 29, 2016
commit b8d6b067379e0f359f2d779bd94524d5993de597
1 change: 1 addition & 0 deletions docs/index.rst
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@
:caption: Vision

vision-usage
vision-annotations
vision-client
vision-color
vision-entity
Expand Down
10 changes: 10 additions & 0 deletions docs/vision-annotations.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
Vision Annotations
==================

Image Annotations
~~~~~~~~~~~~~~~~~

This comment was marked as spam.


.. automodule:: google.cloud.vision.annotations
:members:
:undoc-members:
:show-inheritance:
29 changes: 29 additions & 0 deletions docs/vision-usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,35 @@ or pass in ``credentials`` and ``project`` explicitly.
>>> client = vision.Client(project='my-project', credentials=creds)


Manual Detection
~~~~~~~~~~~~~~~~

You can call the detection method manually.

.. code-block:: python

>>> from google.cloud import vision
>>> from google.cloud.vision.feature import Feature
>>> from google.cloud.vision.feature import FeatureTypes
>>> client = vision.Client()
>>> image = client.image(source_uri='gs://my-test-bucket/image.jpg')
>>> features = [Feature(FeatureTypes.FACE_DETECTION, 5),
... Feature(FeatureTypes.LOGO_DETECTION, 3)]
>>> annotations = image.detect(features)
>>> len(annotations.faces)
2
>>> for face in annotations.faces:
... print(face.joy_likelihood)
0.94099093
0.54453093
>>> len(annotations.logos)
2
>>> for logo in annotations.logos:
... print(logo.description)
'google'
'github'


Face Detection
~~~~~~~~~~~~~~

Expand Down
130 changes: 130 additions & 0 deletions vision/google/cloud/vision/annotations.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Annotations management for Vision API responses."""


from google.cloud.vision.entity import EntityAnnotation
from google.cloud.vision.face import Face
from google.cloud.vision.feature import FeatureTypes
from google.cloud.vision.color import ImagePropertiesAnnotation

This comment was marked as spam.

from google.cloud.vision.safe import SafeSearchAnnotation


_REVERSE_TYPES = {
FeatureTypes.FACE_DETECTION: 'faceAnnotations',
FeatureTypes.IMAGE_PROPERTIES: 'imagePropertiesAnnotation',
FeatureTypes.LABEL_DETECTION: 'labelAnnotations',
FeatureTypes.LANDMARK_DETECTION: 'landmarkAnnotations',
FeatureTypes.LOGO_DETECTION: 'logoAnnotations',
FeatureTypes.SAFE_SEARCH_DETECTION: 'safeSearchAnnotation',
FeatureTypes.TEXT_DETECTION: 'textAnnotations',
}


class Annotations(object):
"""Annotation class for managing responses.

This comment was marked as spam.


:type faces: list
:param faces: List of :class:`~google.cloud.vision.face.Face`.

:type properties: list
:param properties:
List of :class:`~google.cloud.vision.color.ImagePropertiesAnnotation`.

:type labels: list
:param labels: List of
:class:`~google.cloud.vision.entity.EntityAnnotation`.

:type landmarks: list
:param landmarks: List of
:class:`~google.cloud.vision.entity.EntityAnnotation.`

:type logos: list
:param logos: List of
:class:`~google.cloud.vision.entity.EntityAnnotation`.

:type safe_searches: list
:param safe_searches:
List of :class:`~google.cloud.vision.safe.SafeSearchAnnotation`

:type texts: list
:param texts: List of
:class:`~google.cloud.vision.entity.EntityAnnotation`.
"""
def __init__(self, faces=None, properties=None, labels=None,
landmarks=None, logos=None, safe_searches=None, texts=None):
self.faces = faces or []

This comment was marked as spam.

self.properties = properties or []
self.labels = labels or []
self.landmarks = landmarks or []
self.logos = logos or []
self.safe_searches = safe_searches or []
self.texts = texts or []

@classmethod
def from_api_repr(cls, response):
"""Factory: construct an instance of ``Annotations`` from a response.

:type response: dict
:param response: Vision API response object.

:rtype: :class:`~google.cloud.vision.annotations.Annotations`
:returns: An instance of ``Annotations`` with detection types loaded.
"""
annotations = {}

for feature_type in response.keys():
annotations[feature_type] = []

This comment was marked as spam.


for feature_type, annotation in response.items():
annotations[feature_type].extend(
_entity_from_response_type(feature_type, annotation))

This comment was marked as spam.


faces = annotations.get(
_REVERSE_TYPES[FeatureTypes.FACE_DETECTION], [])
properties = annotations.get(
_REVERSE_TYPES[FeatureTypes.IMAGE_PROPERTIES], [])
labels = annotations.get(
_REVERSE_TYPES[FeatureTypes.LABEL_DETECTION], [])
landmarks = annotations.get(
_REVERSE_TYPES[FeatureTypes.LANDMARK_DETECTION], [])
logos = annotations.get(
_REVERSE_TYPES[FeatureTypes.LOGO_DETECTION], [])
safe_searches = annotations.get(
_REVERSE_TYPES[FeatureTypes.SAFE_SEARCH_DETECTION], [])
texts = annotations.get(
_REVERSE_TYPES[FeatureTypes.TEXT_DETECTION], [])

return cls(faces=faces, properties=properties, labels=labels,

This comment was marked as spam.

landmarks=landmarks, logos=logos,
safe_searches=safe_searches, texts=texts)


def _entity_from_response_type(feature_type, results):
"""Convert a JSON result to an entity type based on the feature."""

This comment was marked as spam.


This comment was marked as spam.

detected_objects = []
if feature_type == _REVERSE_TYPES[FeatureTypes.FACE_DETECTION]:

This comment was marked as spam.

detected_objects.extend(
Face.from_api_repr(face) for face in results)
elif feature_type == _REVERSE_TYPES[FeatureTypes.IMAGE_PROPERTIES]:
detected_objects.append(
ImagePropertiesAnnotation.from_api_repr(results))
elif feature_type == _REVERSE_TYPES[FeatureTypes.SAFE_SEARCH_DETECTION]:
detected_objects.append(SafeSearchAnnotation.from_api_repr(results))
else:
for result in results:
detected_objects.append(EntityAnnotation.from_api_repr(result))
return detected_objects
89 changes: 30 additions & 59 deletions vision/google/cloud/vision/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,31 +19,9 @@

from google.cloud._helpers import _to_bytes
from google.cloud._helpers import _bytes_to_unicode
from google.cloud.vision.entity import EntityAnnotation
from google.cloud.vision.face import Face
from google.cloud.vision.annotations import Annotations
from google.cloud.vision.feature import Feature
from google.cloud.vision.feature import FeatureTypes
from google.cloud.vision.color import ImagePropertiesAnnotation
from google.cloud.vision.safe import SafeSearchAnnotation


_FACE_DETECTION = 'FACE_DETECTION'
_IMAGE_PROPERTIES = 'IMAGE_PROPERTIES'
_LABEL_DETECTION = 'LABEL_DETECTION'
_LANDMARK_DETECTION = 'LANDMARK_DETECTION'
_LOGO_DETECTION = 'LOGO_DETECTION'
_SAFE_SEARCH_DETECTION = 'SAFE_SEARCH_DETECTION'
_TEXT_DETECTION = 'TEXT_DETECTION'

_REVERSE_TYPES = {
_FACE_DETECTION: 'faceAnnotations',
_IMAGE_PROPERTIES: 'imagePropertiesAnnotation',
_LABEL_DETECTION: 'labelAnnotations',
_LANDMARK_DETECTION: 'landmarkAnnotations',
_LOGO_DETECTION: 'logoAnnotations',
_SAFE_SEARCH_DETECTION: 'safeSearchAnnotation',
_TEXT_DETECTION: 'textAnnotations',
}


class Image(object):
Expand Down Expand Up @@ -105,7 +83,7 @@ def source(self):
return self._source

def _detect_annotation(self, features):
"""Generic method for detecting a single annotation.
"""Generic method for detecting annotations.
:type features: list
:param features: List of :class:`~google.cloud.vision.feature.Feature`
Expand All @@ -118,12 +96,21 @@ def _detect_annotation(self, features):
:class:`~google.cloud.vision.color.ImagePropertiesAnnotation`,
:class:`~google.cloud.vision.sage.SafeSearchAnnotation`,
"""
detected_objects = []
results = self.client.annotate(self, features)
for feature in features:
detected_objects.extend(
_entity_from_response_type(feature.feature_type, results))
return detected_objects
return Annotations.from_api_repr(results)

def detect(self, features):
"""Detect multiple feature types.
:type features: list of :class:`~google.cloud.vision.feature.Feature`
:param features: List of the ``Feature`` indication the type of
annotation to perform.
:rtype: list
:returns: List of
:class:`~google.cloud.vision.entity.EntityAnnotation`.
"""
return self._detect_annotation(features)

def detect_faces(self, limit=10):
"""Detect faces in image.
Expand All @@ -135,7 +122,8 @@ def detect_faces(self, limit=10):
:returns: List of :class:`~google.cloud.vision.face.Face`.
"""
features = [Feature(FeatureTypes.FACE_DETECTION, limit)]
return self._detect_annotation(features)
annotations = self._detect_annotation(features)
return annotations.faces

def detect_labels(self, limit=10):
"""Detect labels that describe objects in an image.
Expand All @@ -147,7 +135,8 @@ def detect_labels(self, limit=10):
:returns: List of :class:`~google.cloud.vision.entity.EntityAnnotation`
"""
features = [Feature(FeatureTypes.LABEL_DETECTION, limit)]
return self._detect_annotation(features)
annotations = self._detect_annotation(features)
return annotations.labels

def detect_landmarks(self, limit=10):
"""Detect landmarks in an image.
Expand All @@ -160,7 +149,8 @@ def detect_landmarks(self, limit=10):
:class:`~google.cloud.vision.entity.EntityAnnotation`.
"""
features = [Feature(FeatureTypes.LANDMARK_DETECTION, limit)]
return self._detect_annotation(features)
annotations = self._detect_annotation(features)
return annotations.landmarks

def detect_logos(self, limit=10):
"""Detect logos in an image.
Expand All @@ -173,7 +163,8 @@ def detect_logos(self, limit=10):
:class:`~google.cloud.vision.entity.EntityAnnotation`.
"""
features = [Feature(FeatureTypes.LOGO_DETECTION, limit)]
return self._detect_annotation(features)
annotations = self._detect_annotation(features)
return annotations.logos

def detect_properties(self, limit=10):
"""Detect the color properties of an image.
Expand All @@ -186,7 +177,8 @@ def detect_properties(self, limit=10):
:class:`~google.cloud.vision.color.ImagePropertiesAnnotation`.
"""
features = [Feature(FeatureTypes.IMAGE_PROPERTIES, limit)]
return self._detect_annotation(features)
annotations = self._detect_annotation(features)
return annotations.properties

def detect_safe_search(self, limit=10):
"""Retreive safe search properties from an image.
Expand All @@ -199,7 +191,8 @@ def detect_safe_search(self, limit=10):
:class:`~google.cloud.vision.sage.SafeSearchAnnotation`.
"""
features = [Feature(FeatureTypes.SAFE_SEARCH_DETECTION, limit)]
return self._detect_annotation(features)
annotations = self._detect_annotation(features)
return annotations.safe_searches

def detect_text(self, limit=10):
"""Detect text in an image.
Expand All @@ -212,27 +205,5 @@ def detect_text(self, limit=10):
:class:`~google.cloud.vision.entity.EntityAnnotation`.
"""
features = [Feature(FeatureTypes.TEXT_DETECTION, limit)]
return self._detect_annotation(features)


def _entity_from_response_type(feature_type, results):
"""Convert a JSON result to an entity type based on the feature."""
feature_key = _REVERSE_TYPES[feature_type]
annotations = results.get(feature_key, ())
if not annotations:
return []

detected_objects = []
if feature_type == _FACE_DETECTION:
detected_objects.extend(
Face.from_api_repr(face) for face in annotations)
elif feature_type == _IMAGE_PROPERTIES:
detected_objects.append(
ImagePropertiesAnnotation.from_api_repr(annotations))
elif feature_type == _SAFE_SEARCH_DETECTION:
detected_objects.append(
SafeSearchAnnotation.from_api_repr(annotations))
else:
for result in annotations:
detected_objects.append(EntityAnnotation.from_api_repr(result))
return detected_objects
annotations = self._detect_annotation(features)
return annotations.texts
45 changes: 45 additions & 0 deletions vision/unit_tests/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,51 @@ def test_image_with_client(self):
image = client.image(source_uri=IMAGE_SOURCE)
self.assertIsInstance(image, Image)

def test_multiple_detection_from_content(self):
from google.cloud.vision.feature import Feature
from google.cloud.vision.feature import FeatureTypes
from unit_tests._fixtures import LABEL_DETECTION_RESPONSE
from unit_tests._fixtures import LOGO_DETECTION_RESPONSE
RETURNED = LABEL_DETECTION_RESPONSE

This comment was marked as spam.

LOGOS = LOGO_DETECTION_RESPONSE['responses'][0]['logoAnnotations']
RETURNED['responses'][0]['logoAnnotations'] = LOGOS

This comment was marked as spam.


credentials = _Credentials()
client = self._make_one(project=PROJECT, credentials=credentials)
client._connection = _Connection(RETURNED)

limit = 2
label_feature = Feature(FeatureTypes.LABEL_DETECTION, limit)
logo_feature = Feature(FeatureTypes.LOGO_DETECTION, limit)
features = [label_feature, logo_feature]
image = client.image(content=IMAGE_CONTENT)
items = image.detect(features)

self.assertEqual(len(items.logos), 2)
self.assertEqual(len(items.labels), 3)
self.assertEqual(items.logos[0].description, 'Brand1')
self.assertEqual(items.logos[0].score, 0.63192177)
self.assertEqual(items.logos[1].description, 'Brand2')
self.assertEqual(items.logos[1].score, 0.5492993)

self.assertEqual(items.labels[0].description, 'automobile')
self.assertEqual(items.labels[0].score, 0.9776855)
self.assertEqual(items.labels[1].description, 'vehicle')
self.assertEqual(items.labels[1].score, 0.947987)
self.assertEqual(items.labels[2].description, 'truck')
self.assertEqual(items.labels[2].score, 0.88429511)

image_request = client._connection._requested[0]['data']['requests'][0]

This comment was marked as spam.

label_request = image_request['features'][0]
logo_request = image_request['features'][1]

self.assertEqual(B64_IMAGE_CONTENT,
image_request['image']['content'])
self.assertEqual(label_request['maxResults'], 2)
self.assertEqual(label_request['type'], 'LABEL_DETECTION')
self.assertEqual(logo_request['maxResults'], 2)
self.assertEqual(logo_request['type'], 'LOGO_DETECTION')

def test_face_detection_from_source(self):
from google.cloud.vision.face import Face
from unit_tests._fixtures import FACE_DETECTION_RESPONSE
Expand Down