diff --git a/keras_fsl/dataframe/operators/__init__.py b/keras_fsl/dataframe/operators/__init__.py index 99d0d15..4282758 100644 --- a/keras_fsl/dataframe/operators/__init__.py +++ b/keras_fsl/dataframe/operators/__init__.py @@ -4,7 +4,7 @@ __all__ = [ - 'NaiveMaxProba', - 'RandomAssignment', - 'ToKShotDataset', + "NaiveMaxProba", + "RandomAssignment", + "ToKShotDataset", ] diff --git a/keras_fsl/dataframe/operators/to_k_shot_dataset.py b/keras_fsl/dataframe/operators/to_k_shot_dataset.py index a53bc8c..b22b796 100644 --- a/keras_fsl/dataframe/operators/to_k_shot_dataset.py +++ b/keras_fsl/dataframe/operators/to_k_shot_dataset.py @@ -9,7 +9,7 @@ class ToKShotDataset(AbstractOperator): Create tf.data.Dataset with random groups of k_shot consecutive images with the same label """ - def __init__(self, k_shot, preprocessing, label_column='label_one_hot'): + def __init__(self, k_shot, preprocessing, label_column="label_one_hot"): """ Args: @@ -29,16 +29,12 @@ def load_img(annotation): Returns: dict: the input dict with an extra image key. """ - return ( - { - 'image': tf.io.decode_and_crop_jpeg( - tf.io.read_file(annotation['image_name']), - crop_window=annotation['crop_window'], - channels=3, - ), - **annotation, - } - ) + return { + "image": tf.io.decode_and_crop_jpeg( + tf.io.read_file(annotation["image_name"]), crop_window=annotation["crop_window"], channels=3, + ), + **annotation, + } def repeat_k_shot(self, index): return tf.data.Dataset.from_tensors(index).repeat(self.k_shot) @@ -48,7 +44,7 @@ def to_dataset(self, group): Transform a pd.DataFrame into a tf.data.Dataset and load images """ return ( - tf.data.Dataset.from_tensor_slices(group.to_dict('list')) + tf.data.Dataset.from_tensor_slices(group.to_dict("list")) .map(self.load_img, num_parallel_calls=tf.data.experimental.AUTOTUNE) .cache() .shuffle(buffer_size=len(group), reshuffle_each_iteration=True) @@ -56,25 +52,24 @@ def to_dataset(self, group): ) def __call__(self, input_dataframe): - return ( - tf.data.experimental.choose_from_datasets( - datasets=( - input_dataframe - .assign( - label_one_hot=lambda df: pd.get_dummies(df.label).values.tolist(), - crop_window=lambda df: df[["crop_y", "crop_x", "crop_height", "crop_width"]].values.tolist(), - ) - .groupby('label') - .apply(self.to_dataset) - ), - choice_dataset=( - tf.data.Dataset.range(len(input_dataframe.label.unique())) - .shuffle(buffer_size=len(input_dataframe.label.unique()), reshuffle_each_iteration=True) - .flat_map(self.repeat_k_shot) - ), - ) - .map( - lambda annotation: (self.preprocessing(annotation['image']), tf.cast(annotation[self.label_column], tf.float32)), - num_parallel_calls=tf.data.experimental.AUTOTUNE, - ) + return tf.data.experimental.choose_from_datasets( + datasets=( + input_dataframe.assign( + label_one_hot=lambda df: pd.get_dummies(df.label).values.tolist(), + crop_window=lambda df: df[["crop_y", "crop_x", "crop_height", "crop_width"]].values.tolist(), + ) + .groupby("label") + .apply(self.to_dataset) + ), + choice_dataset=( + tf.data.Dataset.range(len(input_dataframe.label.unique())) + .shuffle(buffer_size=len(input_dataframe.label.unique()), reshuffle_each_iteration=True) + .flat_map(self.repeat_k_shot) + ), + ).map( + lambda annotation: ( + self.preprocessing(annotation["image"]), + tf.cast(annotation[self.label_column], tf.float32), + ), + num_parallel_calls=tf.data.experimental.AUTOTUNE, ) diff --git a/keras_fsl/datasets/omniglot.py b/keras_fsl/datasets/omniglot.py index 73999f7..b12bd6a 100644 --- a/keras_fsl/datasets/omniglot.py +++ b/keras_fsl/datasets/omniglot.py @@ -4,35 +4,31 @@ import pandas as pd from tensorflow.keras.utils.data_utils import get_file -BASE_PATH = 'https://raw.githubusercontent.com/brendenlake/omniglot/master/python' +BASE_PATH = "https://raw.githubusercontent.com/brendenlake/omniglot/master/python" def load_dataframe(dataset_name): dataset_path = get_file( - f'{dataset_name}.zip', - origin=f'{BASE_PATH}/{dataset_name}.zip', + f"{dataset_name}.zip", + origin=f"{BASE_PATH}/{dataset_name}.zip", extract=True, - cache_subdir=Path('datasets') / 'omniglot' + cache_subdir=Path("datasets") / "omniglot", ) dataset_dir = os.path.splitext(dataset_path)[0] - dataset = pd.DataFrame(columns=['image_name', 'alphabet', 'label']) + dataset = pd.DataFrame(columns=["image_name", "alphabet", "label"]) for root, _, files in os.walk(dataset_dir): if files: alphabet, label = Path(root).relative_to(dataset_dir).parts root = Path(root) image_names = [root / file for file in files] - dataset = ( - dataset - .append( - pd.DataFrame({'image_name': image_names, 'alphabet': alphabet, 'label': label}), - ignore_index=True, - ) + dataset = dataset.append( + pd.DataFrame({"image_name": image_names, "alphabet": alphabet, "label": label}), ignore_index=True, ) return dataset def load_data(): - train_set = load_dataframe('images_background') - test_set = load_dataframe('images_evaluation') + train_set = load_dataframe("images_background") + test_set = load_dataframe("images_evaluation") return train_set, test_set diff --git a/keras_fsl/losses/yolo_loss.py b/keras_fsl/losses/yolo_loss.py index 9c89f06..ecda644 100644 --- a/keras_fsl/losses/yolo_loss.py +++ b/keras_fsl/losses/yolo_loss.py @@ -24,12 +24,13 @@ def _yolo_loss(y_true, y_pred): for image, pred in zip(y_true, y_pred): loss_objectness.assign_add( - tf.math.reduce_sum(tf.keras.backend.binary_crossentropy(tf.zeros_like(y_pred[..., 4]), y_pred[..., 4]))) + tf.math.reduce_sum(tf.keras.backend.binary_crossentropy(tf.zeros_like(y_pred[..., 4]), y_pred[..., 4])) + ) for box in image: if box[4] < 1: continue - height_width_min = tf.minimum(box[2:4], anchors[['height', 'width']].values) - height_width_max = tf.maximum(box[2:4], anchors[['height', 'width']].values) + height_width_min = tf.minimum(box[2:4], anchors[["height", "width"]].values) + height_width_max = tf.maximum(box[2:4], anchors[["height", "width"]].values) intersection = tf.reduce_prod(height_width_min, axis=-1) union = tf.reduce_prod(height_width_max, axis=-1) iou = intersection / union @@ -46,7 +47,9 @@ def _yolo_loss(y_true, y_pred): loss_objectness.assign_add(tf.keras.backend.binary_crossentropy(box[4], selected_pred[4])) loss_coordinates.assign_add(tf.norm(box[:2] - selected_pred[:2], ord=2)) loss_box.assign_add(tf.norm(box[2:4] - selected_pred[2:4], ord=2)) - loss_classes.assign_add(tf.reduce_sum(tf.keras.backend.binary_crossentropy(box[5:], selected_pred[5:-1]))) + loss_classes.assign_add( + tf.reduce_sum(tf.keras.backend.binary_crossentropy(box[5:], selected_pred[5:-1])) + ) return loss_coordinates + loss_box + loss_objectness + loss_classes diff --git a/keras_fsl/models/activations/__init__.py b/keras_fsl/models/activations/__init__.py index e3134f5..c2199f8 100644 --- a/keras_fsl/models/activations/__init__.py +++ b/keras_fsl/models/activations/__init__.py @@ -2,6 +2,6 @@ from .yolo_coordinates import YoloCoordinates __all__ = [ - 'YoloBox', - 'YoloCoordinates', + "YoloBox", + "YoloCoordinates", ] diff --git a/keras_fsl/models/activations/yolo_box.py b/keras_fsl/models/activations/yolo_box.py index fa76d5b..0f34b5d 100644 --- a/keras_fsl/models/activations/yolo_box.py +++ b/keras_fsl/models/activations/yolo_box.py @@ -15,9 +15,13 @@ def YoloBox(anchor): anchor (Union[pandas.Series, collections.namedtuple]): with key width and height. Note that given a tensor with shape (batch_size, i, j, channels), i is related to height and j to width """ - return Sequential([ - Activation('exponential'), - Lambda(lambda input_, anchor_=anchor: ( - input_ * tf.convert_to_tensor([anchor_.height, anchor_.width], dtype=tf.float32) - )), - ]) + return Sequential( + [ + Activation("exponential"), + Lambda( + lambda input_, anchor_=anchor: ( + input_ * tf.convert_to_tensor([anchor_.height, anchor_.width], dtype=tf.float32) + ) + ), + ] + ) diff --git a/keras_fsl/models/activations/yolo_coordinates.py b/keras_fsl/models/activations/yolo_coordinates.py index e597c8f..4888bf4 100644 --- a/keras_fsl/models/activations/yolo_coordinates.py +++ b/keras_fsl/models/activations/yolo_coordinates.py @@ -27,8 +27,13 @@ def YoloCoordinates(): Activation function for the box center coordinates regression. Coordinates are relative to the image dimension, ie. between 0 and 1 """ - return Sequential([ - Activation('sigmoid'), - Lambda(lambda input_: input_ + tf.cast(tf.expand_dims(build_grid_coordinates(tf.shape(input_)[1:3]), 0), input_.dtype)), - Lambda(lambda input_: input_ / tf.cast(tf.shape(input_)[1:3], input_.dtype)), - ]) + return Sequential( + [ + Activation("sigmoid"), + Lambda( + lambda input_: input_ + + tf.cast(tf.expand_dims(build_grid_coordinates(tf.shape(input_)[1:3]), 0), input_.dtype) + ), + Lambda(lambda input_: input_ / tf.cast(tf.shape(input_)[1:3], input_.dtype)), + ] + ) diff --git a/keras_fsl/models/branch_models/darknet.py b/keras_fsl/models/branch_models/darknet.py index 99ceccf..0eb5789 100644 --- a/keras_fsl/models/branch_models/darknet.py +++ b/keras_fsl/models/branch_models/darknet.py @@ -14,11 +14,7 @@ def conv_2d(*args, **kwargs): @wraps(Conv2D) def conv_block(*args, **kwargs): - return Sequential([ - conv_2d(*args, **kwargs, use_bias=False), - BatchNormalization(), - LeakyReLU(alpha=0.1), - ]) + return Sequential([conv_2d(*args, **kwargs, use_bias=False), BatchNormalization(), LeakyReLU(alpha=0.1),]) def residual_block(input_shape, num_filters, num_blocks): diff --git a/keras_fsl/models/feature_pyramid_net.py b/keras_fsl/models/feature_pyramid_net.py index c5d32f4..09af88a 100644 --- a/keras_fsl/models/feature_pyramid_net.py +++ b/keras_fsl/models/feature_pyramid_net.py @@ -7,59 +7,53 @@ from keras_fsl.models import branch_models, activations -ANCHORS = pd.DataFrame([ - [0, 116 / 416, 90 / 416], - [0, 156 / 416, 198 / 416], - [0, 373 / 416, 326 / 416], - [1, 30 / 416, 61 / 416], - [1, 62 / 416, 45 / 416], - [1, 59 / 416, 119 / 416], - [2, 10 / 416, 13 / 416], - [2, 16 / 416, 30 / 416], - [2, 33 / 416, 23 / 416], -], columns=['scale', 'width', 'height']) +ANCHORS = pd.DataFrame( + [ + [0, 116 / 416, 90 / 416], + [0, 156 / 416, 198 / 416], + [0, 373 / 416, 326 / 416], + [1, 30 / 416, 61 / 416], + [1, 62 / 416, 45 / 416], + [1, 59 / 416, 119 / 416], + [2, 10 / 416, 13 / 416], + [2, 16 / 416, 30 / 416], + [2, 33 / 416, 23 / 416], + ], + columns=["scale", "width", "height"], +) @wraps(Conv2D) def conv_block(*args, **kwargs): - return Sequential([ - Conv2D(*args, **kwargs, use_bias=False), - BatchNormalization(), - ReLU(), - ]) + return Sequential([Conv2D(*args, **kwargs, use_bias=False), BatchNormalization(), ReLU(),]) def bottleneck(filters, *args, **kwargs): - return Sequential([ - conv_block(filters // 4, (1, 1), padding='same'), - conv_block(filters, (3, 3), padding='same'), - ], *args, **kwargs) + return Sequential( + [conv_block(filters // 4, (1, 1), padding="same"), conv_block(filters, (3, 3), padding="same"),], + *args, + **kwargs, + ) def up_sampling_block(filters, *args, **kwargs): - return Sequential([ - conv_block(filters, (1, 1), padding='same'), - UpSampling2D(2), - ], *args, **kwargs) + return Sequential([conv_block(filters, (1, 1), padding="same"), UpSampling2D(2),], *args, **kwargs) def regression_block(activation, *args, **kwargs): - return Sequential([ - Conv2D(2, (1, 1)), - getattr(activations, activation)(*args), - ], **kwargs) + return Sequential([Conv2D(2, (1, 1)), getattr(activations, activation)(*args),], **kwargs) def FeaturePyramidNet( - backbone='MobileNet', + backbone="MobileNet", *args, feature_maps=3, objectness=True, anchors=None, classes=None, weights=None, - coordinates_activation='YoloCoordinates', - box_activation='YoloBox', + coordinates_activation="YoloCoordinates", + box_activation="YoloBox", **kwargs, ): """ @@ -89,19 +83,20 @@ def FeaturePyramidNet( """ if not isinstance(backbone, Model): if isinstance(backbone, str): - backbone = {'name': backbone, 'init': {'include_top': False, 'input_shape': (416, 416, 3)}} - backbone_name = backbone['name'] - backbone = getattr(branch_models, backbone_name)(**backbone.get('init', {})) + backbone = {"name": backbone, "init": {"include_top": False, "input_shape": (416, 416, 3)}} + backbone_name = backbone["name"] + backbone = getattr(branch_models, backbone_name)(**backbone.get("init", {})) output_shapes = ( - pd.DataFrame([ - layer.input_shape[0] - if isinstance(layer.input_shape, list) - else layer.output_shape - for layer in backbone.layers - ], columns=['batch_size', 'height', 'width', 'channels']) + pd.DataFrame( + [ + layer.input_shape[0] if isinstance(layer.input_shape, list) else layer.output_shape + for layer in backbone.layers + ], + columns=["batch_size", "height", "width", "channels"], + ) .loc[lambda df: df.width.iloc[0] % df.width == 0] - .drop_duplicates(['width', 'height'], keep='last') + .drop_duplicates(["width", "height"], keep="last") .sort_index(ascending=False) ) @@ -109,29 +104,50 @@ def FeaturePyramidNet( for output_shape in output_shapes.iloc[:feature_maps].itertuples(): input_ = backbone.layers[output_shape.Index].output if outputs: - pyramid_input = up_sampling_block(output_shape.channels, name=f'up_sampling_{output_shape.channels}')(outputs[-1]) + pyramid_input = up_sampling_block(output_shape.channels, name=f"up_sampling_{output_shape.channels}")( + outputs[-1] + ) input_ = Concatenate()([input_, pyramid_input]) - outputs += [bottleneck(output_shape.channels, name=f'bottleneck_{output_shape.channels}')(input_)] + outputs += [bottleneck(output_shape.channels, name=f"bottleneck_{output_shape.channels}")(input_)] if classes is not None: if anchors is None: anchors = ANCHORS.copy().round(3) - anchors = anchors.assign(id=lambda df: 'scale_' + df.scale.astype(str) + '_' + df.width.astype(str) + 'x' + df.height.astype(str)) + anchors = anchors.assign( + id=lambda df: "scale_" + df.scale.astype(str) + "_" + df.width.astype(str) + "x" + df.height.astype(str) + ) outputs = [ - Reshape((-1, 4 + int(objectness) + len(classes)))(Concatenate(axis=3, name=f'anchor_{anchor.id}_output')( - [regression_block(coordinates_activation, name=f'{anchor.id}_box_yx')(outputs[anchor.scale])] + - [regression_block(box_activation, anchor, name=f'{anchor.id}_box_hw')(outputs[anchor.scale])] + - ([Conv2D(1, (1, 1), name=f'{anchor.id}_objectness', activation='sigmoid')(outputs[anchor.scale])] if objectness else []) + - [Conv2D(1, (1, 1), name=f'{anchor.id}_{label}', activation='sigmoid')(outputs[anchor.scale]) for label in classes] - )) + Reshape((-1, 4 + int(objectness) + len(classes)))( + Concatenate(axis=3, name=f"anchor_{anchor.id}_output")( + [regression_block(coordinates_activation, name=f"{anchor.id}_box_yx")(outputs[anchor.scale])] + + [regression_block(box_activation, anchor, name=f"{anchor.id}_box_hw")(outputs[anchor.scale])] + + ( + [Conv2D(1, (1, 1), name=f"{anchor.id}_objectness", activation="sigmoid")(outputs[anchor.scale])] + if objectness + else [] + ) + + [ + Conv2D(1, (1, 1), name=f"{anchor.id}_{label}", activation="sigmoid")(outputs[anchor.scale]) + for label in classes + ] + ) + ) for anchor in anchors.itertuples() ] - outputs = Concatenate(axis=1)([ - Lambda(lambda output, index_=index: ( - tf.concat([output, tf.expand_dims(tf.ones(tf.shape(output)[:2], dtype=output.dtype) * index_, -1)], axis=-1) - ))(outputs[index]) - for index, anchor in anchors.iterrows() - ]) + # noinspection PyTypeChecker + outputs = Concatenate(axis=1)( + [ + Lambda( + lambda output, index_=index: ( + tf.concat( + [output, tf.expand_dims(tf.ones(tf.shape(output)[:2], dtype=output.dtype) * index_, -1)], + axis=-1, + ) + ) + )(outputs[index]) + for index, anchor in anchors.iterrows() + ] + ) model = Model(backbone.input, outputs, *args, **kwargs) if weights is not None: diff --git a/notebooks/batch_gram_matrix_training.py b/notebooks/batch_gram_matrix_training.py index 8740cc6..f3f885e 100644 --- a/notebooks/batch_gram_matrix_training.py +++ b/notebooks/batch_gram_matrix_training.py @@ -86,9 +86,7 @@ ) train_val_test_split = { - day: key - for key, days in yaml.safe_load(open("data/annotations/train_val_test_split.yaml")).items() - for day in days + day: key for key, days in yaml.safe_load(open("data/annotations/train_val_test_split.yaml")).items() for day in days } all_annotations = ( pd.read_csv("data/annotations/cropped_images.csv") @@ -109,15 +107,8 @@ optimizer = Adam(lr=1e-4) margin = 0.05 batch_size = 64 -datasets = ( - all_annotations - .groupby('split') - .apply(lambda group: ( - group - .pipe(ToKShotDataset(k_shot=8, preprocessing=preprocessing)) - .batch(batch_size) - .repeat() - )) +datasets = all_annotations.groupby("split").apply( + lambda group: (group.pipe(ToKShotDataset(k_shot=8, preprocessing=preprocessing)).batch(batch_size).repeat()) ) model.compile( optimizer=optimizer, @@ -125,10 +116,10 @@ metrics=[binary_crossentropy(0.0), accuracy(margin), mean_score_classification_loss, min_eigenvalue], ) model.fit( - datasets['train'], - steps_per_epoch=all_annotations.split.value_counts()['train'] // batch_size, - validation_data=datasets['val'], - validation_steps=all_annotations.split.value_counts()['val'] // batch_size, + datasets["train"], + steps_per_epoch=all_annotations.split.value_counts()["train"] // batch_size, + validation_data=datasets["val"], + validation_steps=all_annotations.split.value_counts()["val"] // batch_size, initial_epoch=0, epochs=5, callbacks=callbacks, @@ -142,10 +133,10 @@ metrics=[binary_crossentropy(0.0), accuracy(margin), mean_score_classification_loss, min_eigenvalue], ) model.fit( - datasets['train'], - steps_per_epoch=all_annotations.split.value_counts()['train'] // batch_size, - validation_data=datasets['val'], - validation_steps=all_annotations.split.value_counts()['val'] // batch_size, + datasets["train"], + steps_per_epoch=all_annotations.split.value_counts()["train"] // batch_size, + validation_data=datasets["val"], + validation_steps=all_annotations.split.value_counts()["val"] // batch_size, initial_epoch=5, epochs=20, callbacks=callbacks, @@ -159,12 +150,12 @@ n_way = 5 n_episode = 100 test_dataset = ( - tf.data.Dataset.from_tensor_slices(test_set.to_dict('list')) - .map(lambda annotation: tf.io.decode_and_crop_jpeg( - contents=tf.io.read_file(annotation['image_name']), - crop_window=annotation['crop_window'], - channels=3, - )) + tf.data.Dataset.from_tensor_slices(test_set.to_dict("list")) + .map( + lambda annotation: tf.io.decode_and_crop_jpeg( + contents=tf.io.read_file(annotation["image_name"]), crop_window=annotation["crop_window"], channels=3, + ) + ) .map(preprocessing) .batch(64) ) @@ -175,17 +166,14 @@ for _ in range(n_episode): selected_labels = np.random.choice(test_set.label.unique(), size=n_way, replace=True) support_set = ( - test_set - .loc[lambda df: df.label.isin(selected_labels)] + test_set.loc[lambda df: df.label.isin(selected_labels)] .groupby("label") .apply(lambda group: group.sample(k_shot)) .reset_index("label", drop=True) ) - query_set = ( - test_set - .loc[lambda df: df.label.isin(selected_labels)] - .loc[lambda df: ~df.index.isin(support_set.index)] - ) + query_set = test_set.loc[lambda df: df.label.isin(selected_labels)].loc[ + lambda df: ~df.index.isin(support_set.index) + ] support_set_embeddings = embeddings[support_set.index] query_set_embeddings = embeddings[query_set.index] test_sequence = ProductSequence(