Add files via upload

keras_csp/bbox_process.py

@@ -0,0 +1,271 @@
+from __future__ import division
+import numpy as np
+from nms_wrapper import nms
+
+def parse_det(Y, C, score=0.1, down=4,scale='h'):
+    seman = Y[0][0, :, :, 0]
+    if scale=='h':
+        height = np.exp(Y[1][0, :, :, 0])*down
+        width = 0.41*height
+    elif scale=='w':
+        width = np.exp(Y[1][0, :, :, 0])*down
+        height = width/0.41
+    elif scale=='hw':
+        height = np.exp(Y[1][0, :, :, 0])*down
+        width = np.exp(Y[1][0, :, :, 1])*down
+    y_c, x_c = np.where(seman > score)
+    boxs = []
+    if len(y_c) > 0:
+        for i in range(len(y_c)):
+            h = height[y_c[i], x_c[i]]
+            w = width[y_c[i], x_c[i]]
+            s = seman[y_c[i], x_c[i]]
+            x1, y1 = max(0, (x_c[i]+0.5) * down - w / 2), max(0, (y_c[i]+0.5) * down - h / 2)
+            boxs.append([x1, y1, min(x1 + w, C.size_test[1]), min(y1 + h, C.size_test[0]), s])
+        boxs = np.asarray(boxs, dtype=np.float32)
+        keep = nms(boxs, 0.5, usegpu=False, gpu_id=0)
+        boxs = boxs[keep, :]
+    return boxs
+
+def parse_det_top(Y, C, score=0.1):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    y_c, x_c = np.where(seman > score)
+    boxs = []
+    if len(y_c) > 0:
+        for i in range(len(y_c)):
+            h = np.exp(height[y_c[i], x_c[i]]) * 4
+            w = 0.41 * h
+            s = seman[y_c[i], x_c[i]]
+            x1, y1 = max(0, x_c[i] * 4 + 2 - w / 2), max(0, y_c[i] * 4 + 2)
+            boxs.append([x1, y1, min(x1 + w, C.size_test[1]), min(y1 + h, C.size_test[0]), s])
+        boxs = np.asarray(boxs, dtype=np.float32)
+        keep = nms(boxs, 0.5, usegpu=False, gpu_id=0)
+        boxs = boxs[keep, :]
+    return boxs
+
+def parse_det_bottom(Y, C, score=0.1):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    y_c, x_c = np.where(seman > score)
+    boxs = []
+    if len(y_c) > 0:
+        for i in range(len(y_c)):
+            h = np.exp(height[y_c[i], x_c[i]]) * 4
+            w = 0.41 * h
+            s = seman[y_c[i], x_c[i]]
+            x1, y1 = max(0, x_c[i] * 4 + 2 - w / 2), max(0, y_c[i] * 4 + 2-h)
+            boxs.append([x1, y1, min(x1 + w, C.size_test[1]), min(y1 + h, C.size_test[0]), s])
+        boxs = np.asarray(boxs, dtype=np.float32)
+        keep = nms(boxs, 0.5, usegpu=False, gpu_id=0)
+        boxs = boxs[keep, :]
+    return boxs
+
+def parse_det_offset(Y, C, score=0.1,down=4):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    offset_y = Y[2][0, :, :, 0]
+    offset_x = Y[2][0, :, :, 1]
+    y_c, x_c = np.where(seman > score)
+    boxs = []
+    if len(y_c) > 0:
+        for i in range(len(y_c)):
+            h = np.exp(height[y_c[i], x_c[i]]) * down
+            w = 0.41*h
+            o_y = offset_y[y_c[i], x_c[i]]
+            o_x = offset_x[y_c[i], x_c[i]]
+            s = seman[y_c[i], x_c[i]]
+            x1, y1 = max(0, (x_c[i] + o_x + 0.5) * down - w / 2), max(0, (y_c[i] + o_y + 0.5) * down - h / 2)
+            boxs.append([x1, y1, min(x1 + w, C.size_test[1]), min(y1 + h, C.size_test[0]), s])
+        boxs = np.asarray(boxs, dtype=np.float32)
+        keep = nms(boxs, 0.5, usegpu=False, gpu_id=0)
+        boxs = boxs[keep, :]
+    return boxs
+
+def parse_wider_h_offset(Y, C, score=0.1,down=4,nmsthre=0.5):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    offset_y = Y[2][0, :, :, 0]
+    offset_x = Y[2][0, :, :, 1]
+    y_c, x_c = np.where(seman > score)
+    boxs = []
+    if len(y_c) > 0:
+        for i in range(len(y_c)):
+            h = np.exp(height[y_c[i], x_c[i]]) * down
+            w = h
+            o_y = offset_y[y_c[i], x_c[i]]
+            o_x = offset_x[y_c[i], x_c[i]]
+            s = seman[y_c[i], x_c[i]]
+            x1, y1 = max(0, (x_c[i] + o_x + 0.5) * down - w / 2), max(0, (y_c[i] + o_y + 0.5) * down - h / 2)
+            x1, y1 = min(x1, C.size_test[1]), min(y1, C.size_test[0])
+            boxs.append([x1, y1, min(x1 + w, C.size_test[1]), min(y1 + h, C.size_test[0]), s])
+        boxs = np.asarray(boxs, dtype=np.float32)
+        # keep = nms(boxs, nmsthre, usegpu=True, gpu_id=0)
+        # boxs = boxs[keep, :]
+    boxs = soft_bbox_vote(boxs, thre=nmsthre)
+    return boxs
+
+def parse_shanghai_h_offset(Y, C, score=0.1,down=4,nmsthre=0.5):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    offset_y = Y[2][0, :, :, 0]
+    offset_x = Y[2][0, :, :, 1]
+    y_c, x_c = np.where(seman > score)
+    h = np.exp(height[y_c, x_c]) * down
+    w = h
+    o_y = offset_y[y_c, x_c]
+    o_x = offset_x[y_c, x_c]
+    s = seman[y_c, x_c]
+    x1 = np.clip((x_c + o_x + 0.5) * down - w / 2, 0, C.size_test[1])
+    y1 = np.clip((y_c + o_y + 0.5) * down - h / 2, 0, C.size_test[0])
+    x2 = np.clip(x1 + w, None, C.size_test[1])
+    y2 = np.clip(y1 + h, None, C.size_test[0])
+    boxs = np.concatenate((x1[...,np.newaxis], y1[...,np.newaxis], x2[...,np.newaxis],
+                           y2[...,np.newaxis], s[...,np.newaxis]), axis=1)
+    boxs = np.asarray(boxs, dtype=np.float32)
+    boxs = soft_bbox_vote(boxs, thre=nmsthre, score=score)
+    # keep = nms(boxs, nmsthre, usegpu=True, gpu_id=0)
+    # boxs = boxs[keep, :]
+    return boxs
+
+def parse_shanghai_h_nooff(Y, C, score=0.1,down=4,nmsthre=0.5):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    y_c, x_c = np.where(seman > score)
+    h = np.exp(height[y_c, x_c]) * down
+    w = h
+    s = seman[y_c, x_c]
+    x1 = np.clip(x_c * down - w / 2, 0, C.size_test[1])
+    y1 = np.clip(y_c * down - h / 2, 0, C.size_test[0])
+    x2 = np.clip(x1 + w, None, C.size_test[1])
+    y2 = np.clip(y1 + h, None, C.size_test[0])
+    boxs = np.concatenate((x1[...,np.newaxis], y1[...,np.newaxis], x2[...,np.newaxis],
+                           y2[...,np.newaxis], s[...,np.newaxis]), axis=1)
+    boxs = np.asarray(boxs, dtype=np.float32)
+    boxs = soft_bbox_vote(boxs, thre=nmsthre, score=score)
+    # keep = nms(boxs, nmsthre, usegpu=True, gpu_id=0)
+    # boxs = boxs[keep, :]
+    return boxs
+
+def parse_wider_offset(Y, C, score=0.1,down=4,nmsthre=0.5):
+    seman = Y[0][0, :, :, 0]
+    height = Y[1][0, :, :, 0]
+    width = Y[1][0, :, :, 1]
+    offset_y = Y[2][0, :, :, 0]
+    offset_x = Y[2][0, :, :, 1]
+    y_c, x_c = np.where(seman > score)
+    boxs = []
+    if len(y_c) > 0:
+        for i in range(len(y_c)):
+            h = np.exp(height[y_c[i], x_c[i]]) * down
+            w = np.exp(width[y_c[i], x_c[i]]) * down
+            o_y = offset_y[y_c[i], x_c[i]]
+            o_x = offset_x[y_c[i], x_c[i]]
+            s = seman[y_c[i], x_c[i]]
+            x1, y1 = max(0, (x_c[i] + o_x + 0.5) * down - w / 2), max(0, (y_c[i] + o_y + 0.5) * down - h / 2)
+            x1, y1 = min(x1, C.size_test[1]), min(y1, C.size_test[0])
+            boxs.append([x1, y1, min(x1 + w, C.size_test[1]), min(y1 + h, C.size_test[0]), s])
+        boxs = np.asarray(boxs, dtype=np.float32)
+        #keep = nms(boxs, nmsthre, usegpu=False, gpu_id=0)
+        #boxs = boxs[keep, :]
+	boxs = soft_bbox_vote(boxs,thre=nmsthre)
+    return boxs
+
+def soft_bbox_vote(det,thre=0.35,score=0.05):
+    if isinstance(det, list):
+        return det
+    if det.shape[0] <= 1:
+        return det
+    order = det[:, 4].ravel().argsort()[::-1]
+    det = det[order, :]
+    dets = []
+    while det.shape[0] > 0:
+        # IOU
+        area = (det[:, 2] - det[:, 0] + 1) * (det[:, 3] - det[:, 1] + 1)
+        xx1 = np.maximum(det[0, 0], det[:, 0])
+        yy1 = np.maximum(det[0, 1], det[:, 1])
+        xx2 = np.minimum(det[0, 2], det[:, 2])
+        yy2 = np.minimum(det[0, 3], det[:, 3])
+        w = np.maximum(0.0, xx2 - xx1 + 1)
+        h = np.maximum(0.0, yy2 - yy1 + 1)
+        inter = w * h
+        o = inter / (area[0] + area[:] - inter)
+
+        # get needed merge det and delete these det
+        merge_index = np.where(o >= thre)[0]
+        det_accu = det[merge_index, :]
+        det_accu_iou = o[merge_index]
+        det = np.delete(det, merge_index, 0)
+
+        if merge_index.shape[0] <= 1:
+            try:
+                dets = np.row_stack((dets, det_accu))
+            except:
+                dets = det_accu
+            continue
+        else:
+            soft_det_accu = det_accu.copy()
+            soft_det_accu[:, 4] = soft_det_accu[:, 4] * (1 - det_accu_iou)
+            soft_index = np.where(soft_det_accu[:, 4] >= score)[0]
+            soft_det_accu = soft_det_accu[soft_index, :]
+
+            det_accu[:, 0:4] = det_accu[:, 0:4] * np.tile(det_accu[:, -1:], (1, 4))
+            max_score = np.max(det_accu[:, 4])
+            det_accu_sum = np.zeros((1, 5))
+            det_accu_sum[:, 0:4] = np.sum(det_accu[:, 0:4], axis=0) / np.sum(det_accu[:, -1:])
+            det_accu_sum[:, 4] = max_score
+
+            if soft_det_accu.shape[0] > 0:
+                det_accu_sum = np.row_stack((soft_det_accu, det_accu_sum))
+
+            try:
+                dets = np.row_stack((dets, det_accu_sum))
+            except:
+                dets = det_accu_sum
+
+    order = dets[:, 4].ravel().argsort()[::-1]
+    dets = dets[order, :]
+    return dets
+
+def bbox_vote(det,thre):
+    if det.shape[0] <= 1:
+        return det
+    order = det[:, 4].ravel().argsort()[::-1]
+    det = det[order, :]
+    # det = det[np.where(det[:, 4] > 0.2)[0], :]
+    dets = []
+    while det.shape[0] > 0:
+        # IOU
+        area = (det[:, 2] - det[:, 0] + 1) * (det[:, 3] - det[:, 1] + 1)
+        xx1 = np.maximum(det[0, 0], det[:, 0])
+        yy1 = np.maximum(det[0, 1], det[:, 1])
+        xx2 = np.minimum(det[0, 2], det[:, 2])
+        yy2 = np.minimum(det[0, 3], det[:, 3])
+        w = np.maximum(0.0, xx2 - xx1 + 1)
+        h = np.maximum(0.0, yy2 - yy1 + 1)
+        inter = w * h
+        o = inter / (area[0] + area[:] - inter)
+
+        # get needed merge det and delete these  det
+        merge_index = np.where(o >= thre)[0]
+        det_accu = det[merge_index, :]
+        det = np.delete(det, merge_index, 0)
+
+        if merge_index.shape[0] <= 1:
+            try:
+                dets = np.row_stack((dets, det_accu))
+            except:
+                dets = det_accu
+            continue
+        else:
+            det_accu[:, 0:4] = det_accu[:, 0:4] * np.tile(det_accu[:, -1:], (1, 4))
+            max_score = np.max(det_accu[:, 4])
+            det_accu_sum = np.zeros((1, 5))
+            det_accu_sum[:, 0:4] = np.sum(det_accu[:, 0:4], axis=0) / np.sum(det_accu[:, -1:])
+            det_accu_sum[:, 4] = max_score
+            try:
+                dets = np.row_stack((dets, det_accu_sum))
+            except:
+                dets = det_accu_sum
+
+    return dets

keras_csp/bbox_transform.py

@@ -0,0 +1,91 @@
+# --------------------------------------------------------
+# Fast R-CNN
+# Copyright (c) 2015 Microsoft
+# Licensed under The MIT License [see LICENSE for details]
+# Written by Ross Girshick
+# --------------------------------------------------------
+
+import numpy as np
+
+def bbox_transform(ex_rois, gt_rois):
+    ex_widths = ex_rois[:, 2] - ex_rois[:, 0] + 1.0
+    ex_heights = ex_rois[:, 3] - ex_rois[:, 1] + 1.0
+    ex_ctr_x = ex_rois[:, 0] + 0.5 * ex_widths
+    ex_ctr_y = ex_rois[:, 1] + 0.5 * ex_heights
+
+    gt_widths = gt_rois[:, 2] - gt_rois[:, 0] + 1.0
+    gt_heights = gt_rois[:, 3] - gt_rois[:, 1] + 1.0
+    gt_ctr_x = gt_rois[:, 0] + 0.5 * gt_widths
+    gt_ctr_y = gt_rois[:, 1] + 0.5 * gt_heights
+
+    targets_dx = (gt_ctr_x - ex_ctr_x) / ex_widths
+    targets_dy = (gt_ctr_y - ex_ctr_y) / ex_heights
+    targets_dw = np.log(gt_widths / ex_widths)
+    targets_dh = np.log(gt_heights / ex_heights)
+
+    targets = np.vstack(
+        (targets_dx, targets_dy, targets_dw, targets_dh)).transpose()
+    return targets
+
+def bbox_transform_inv(boxes, deltas):
+    if boxes.shape[0] == 0:
+        return np.zeros((0, deltas.shape[1]), dtype=deltas.dtype)
+
+    boxes = boxes.astype(deltas.dtype, copy=False)
+
+    widths = boxes[:, 2] - boxes[:, 0] + 1.0
+    heights = boxes[:, 3] - boxes[:, 1] + 1.0
+    ctr_x = boxes[:, 0] + 0.5 * widths
+    ctr_y = boxes[:, 1] + 0.5 * heights
+
+    dx = deltas[:, 0::4]
+    dy = deltas[:, 1::4]
+    dw = deltas[:, 2::4]
+    dh = deltas[:, 3::4]
+
+    pred_ctr_x = dx * widths[:, np.newaxis] + ctr_x[:, np.newaxis]
+    pred_ctr_y = dy * heights[:, np.newaxis] + ctr_y[:, np.newaxis]
+    pred_w = np.exp(dw) * widths[:, np.newaxis]
+    pred_h = np.exp(dh) * heights[:, np.newaxis]
+
+    pred_boxes = np.zeros(deltas.shape, dtype=deltas.dtype)
+    # x1
+    pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * pred_w
+    # y1
+    pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * pred_h
+    # x2
+    pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * pred_w
+    # y2
+    pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * pred_h
+
+    return pred_boxes
+
+def compute_targets(ex_rois, gt_rois, classifier_regr_std,std):
+    """Compute bounding-box regression targets for an image."""
+
+    assert ex_rois.shape[0] == gt_rois.shape[0]
+    assert ex_rois.shape[1] == 4
+    assert gt_rois.shape[1] == 4
+
+    targets = bbox_transform(ex_rois, gt_rois)
+	# Optionally normalize targets by a precomputed mean and stdev
+    if std:
+		targets = targets/np.array(classifier_regr_std)
+    return targets
+
+def clip_boxes(boxes, im_shape):
+    # boxes[:, 0::4] = np.maximum(np.minimum(boxes[:, 0::4], im_shape[1] - 1), 0)
+    # # y1 >= 0
+    # boxes[:, 1::4] = np.maximum(np.minimum(boxes[:, 1::4], im_shape[0] - 1), 0)
+    # # x2 < im_shape[1]
+    # boxes[:, 2::4] = np.maximum(np.minimum(boxes[:, 2::4], im_shape[1] - 1), 0)
+    # # y2 < im_shape[0]
+    # boxes[:, 3::4] = np.maximum(np.minimum(boxes[:, 3::4], im_shape[0] - 1), 0)
+    boxes[:, 0] = np.maximum(np.minimum(boxes[:, 0], im_shape[1] - 1), 0)
+    # y1 >= 0
+    boxes[:, 1] = np.maximum(np.minimum(boxes[:, 1], im_shape[0] - 1), 0)
+    # x2 < im_shape[1]
+    boxes[:, 2] = np.maximum(np.minimum(boxes[:, 2], im_shape[1] - 1), 0)
+    # y2 < im_shape[0]
+    boxes[:, 3] = np.maximum(np.minimum(boxes[:, 3], im_shape[0] - 1), 0)
+    return boxes

keras_csp/config.py

@@ -0,0 +1,27 @@
+# default config
+class Config(object):
+	def __init__(self):
+		self.gpu_ids = '0'
+		self.onegpu = 2
+		self.num_epochs = 150
+		self.add_epoch = 0
+		self.iter_per_epoch = 2000
+		self.init_lr = 1e-4
+		self.alpha = 0.999
+
+		# setting for network architechture
+		self.network = 'resnet50' # or 'mobilenet'
+		self.point = 'center' # or 'top', 'bottom
+		self.scale = 'h' # or 'w', 'hw'
+		self.num_scale = 1 # 1 for height (or width) prediction, 2 for height+width prediction
+		self.offset = False # append offset prediction or not
+		self.down = 4 # downsampling rate of the feature map for detection
+		self.radius = 2 # surrounding areas of positives for the scale map
+
+		# setting for data augmentation
+		self.use_horizontal_flips = True
+		self.brightness = (0.5, 2, 0.5)
+		self.size_train = (336, 448)
+
+		# image channel-wise mean to subtract, the order is BGR
+		self.img_channel_mean = [103.939, 116.779, 123.68]