Fix pylint issues. Added pylint checking. (#322)

* Added pylint job for CI

* Rework pip install

* Fixed remaining pylint warnings

Co-authored-by: Andrey Zhavoronkov <andrey.zhavoronkov@intel.com>

.github/workflows/pylint.yml

@@ -0,0 +1,16 @@
+name: Linter
+on: pull_request
+jobs:
+  PyLint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Run checks
+        run: |
+          pip install --user -r <(grep "^pylint" ./requirements.txt)
+          echo "Pylint version: "`pylint --version | head -1`
+          git ls-files -z '*.py' | xargs -0 pylint -r n

datumaro/cli/contexts/project/__init__.py

@@ -8,8 +8,8 @@
 import os
 import os.path as osp
 import shutil
-import numpy as np
 from enum import Enum
+import numpy as np
 
 from datumaro.components.dataset_filter import DatasetItemEncoder
 from datumaro.components.extractor import AnnotationType
@@ -888,4 +888,4 @@ def build_parser(parser_ctor=argparse.ArgumentParser):
     add_subparser(subparsers, 'stats', build_stats_parser)
     add_subparser(subparsers, 'validate', build_validate_parser)
 
-    return parser
+    return parser

datumaro/cli/contexts/project/diff.py

@@ -9,11 +9,10 @@
 import logging as log
 import os
 import os.path as osp
-
+import warnings
 import cv2
 import numpy as np
 
-import warnings
 with warnings.catch_warnings():
     warnings.simplefilter("ignore")
     import tensorboardX as tb

datumaro/components/algorithms/rise.py

@@ -5,8 +5,8 @@
 
 # pylint: disable=unused-variable
 
-import numpy as np
 from math import ceil
+import numpy as np
 
 from datumaro.components.extractor import AnnotationType
 from datumaro.util.annotation_util import nms
@@ -200,4 +200,4 @@ def apply(self, image, progressive=False):
                 if progressive:
                     yield self.normalize_hmaps(heatmaps.copy(), total_counts)
 
-        yield self.normalize_hmaps(heatmaps, total_counts)
+        yield self.normalize_hmaps(heatmaps, total_counts)

datumaro/components/environment.py

@@ -4,12 +4,12 @@
 
 from functools import partial
 from glob import glob
-import git
 import inspect
 import logging as log
 import os
 import os.path as osp
 from typing import Dict, Iterable
+import git
 
 from datumaro.components.config import Config
 from datumaro.components.config_model import Model, Source

datumaro/components/operations.py

@@ -6,12 +6,12 @@
 from copy import deepcopy
 import hashlib
 import logging as log
+from unittest import TestCase
 
 import attr
 import cv2
 import numpy as np
 from attr import attrib, attrs
-from unittest import TestCase
 
 from datumaro.components.cli_plugin import CliPlugin
 from datumaro.util import find, filter_dict
@@ -1508,4 +1508,4 @@ def compare_datasets(self, a, b):
             a_unmatched |= set(a_id for a_id, m in a_matches.items() if not m)
             b_unmatched |= set(b_id for b_id, m in b_matches.items() if not m)
 
-        return matched, unmatched, a_unmatched, b_unmatched, errors
+        return matched, unmatched, a_unmatched, b_unmatched, errors

datumaro/components/project.py

@@ -140,7 +140,7 @@ def categories(self):
     def __len__(self):
         return sum(len(s) for s in self._subsets.values())
 
-    def get(self, id, subset=None, path=None):
+    def get(self, id, subset=None, path=None): # pylint: disable=arguments-differ
         if path:
             source = path[0]
             return self._sources[source].get(id=id, subset=subset)

datumaro/plugins/accuracy_checker_plugin/details/ac.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-from datumaro.util.tf_util import import_tf
+from datumaro.util.tf_util import import_tf # pylint: disable=wrong-import-order
 import_tf() # prevent TF loading and potential interpreter crash
 
 from itertools import groupby

datumaro/plugins/accuracy_checker_plugin/details/representation.py

@@ -3,7 +3,7 @@
 #
 # SPDX-License-Identifier: MIT
 
-from datumaro.util.tf_util import import_tf
+from datumaro.util.tf_util import import_tf # pylint: disable=wrong-import-order
 import_tf() # prevent TF loading and potential interpreter crash
 
 import accuracy_checker.representation as ac

datumaro/plugins/coco_format/converter.py

@@ -5,11 +5,11 @@
 
 import json
 import logging as log
-import numpy as np
 import os
 import os.path as osp
 from enum import Enum, auto
 from itertools import chain, groupby
+import numpy as np
 
 import pycocotools.mask as mask_utils
 

datumaro/plugins/datumaro_format/converter.py

@@ -6,10 +6,11 @@
 # pylint: disable=no-self-use
 
 import json
-import numpy as np
 import os
 import os.path as osp
+import numpy as np
 
+import pycocotools.mask as mask_utils
 from datumaro.components.converter import Converter
 from datumaro.components.dataset import ItemStatus
 from datumaro.components.extractor import (
@@ -18,7 +19,6 @@
     LabelCategories, MaskCategories, PointsCategories
 )
 from datumaro.util import cast
-import pycocotools.mask as mask_utils
 
 from .format import DatumaroPath
 
@@ -255,7 +255,7 @@ def apply(self):
         for subset, writer in subsets.items():
             writer.write(annotations_dir)
 
-    def _save_image(self, item, path=None):
+    def _save_image(self, item, path=None): # pylint: disable=arguments-differ
         super()._save_image(item,
             osp.join(self._images_dir, self._make_image_filename(item)))
 

datumaro/plugins/kitti_format/converter.py

@@ -14,7 +14,7 @@
 from datumaro.components.converter import Converter
 from datumaro.components.extractor import (AnnotationType,
     CompiledMask, LabelCategories)
-from datumaro.util import find, parse_str_enum_value, str_to_bool, cast
+from datumaro.util import parse_str_enum_value, str_to_bool, cast
 from datumaro.util.annotation_util import make_label_id_mapping
 from datumaro.util.image import save_image
 from datumaro.util.mask_tools import paint_mask

datumaro/plugins/labelme_format.py

@@ -3,13 +3,13 @@
 # SPDX-License-Identifier: MIT
 
 from collections import defaultdict
-from defusedxml import ElementTree
 from functools import partial
 from glob import glob, iglob
 import logging as log
-import numpy as np
 import os
 import os.path as osp
+from defusedxml import ElementTree
+import numpy as np
 
 from datumaro.components.extractor import (Extractor, Importer,
     DatasetItem, AnnotationType, Mask, Bbox, Polygon, LabelCategories)
@@ -59,7 +59,7 @@ def _parse(self, dataset_root):
             subset = ''
             if 1 < len(path_parts):
                 subset = path_parts[0]
-                item_path = osp.join(*path_parts[1:])
+                item_path = osp.join(*path_parts[1:]) # pylint: disable=no-value-for-parameter
 
             root = ElementTree.parse(xml_path)
 
@@ -408,8 +408,8 @@ def _save_item(self, item, subset_dir):
                        cast(v, int) is not None and str(int(v)) == v:
                         v = f'"{v}"' # add escaping for string values
                     else:
-                        v = self._escape(v)
-                attrs.append('%s=%s' % (self._escape(k), v))
+                        v = self._escape(v) # pylint: disable=redundant-keyword-arg due FP https://github.com/PyCQA/pylint/issues/2271
+                attrs.append('%s=%s' % (self._escape(k), v)) # pylint: disable=redundant-keyword-arg due FP https://github.com/PyCQA/pylint/issues/2271
             ET.SubElement(obj_elem, 'attributes').text = ', '.join(attrs)
 
             obj_id += 1
@@ -439,4 +439,4 @@ def _save_item(self, item, subset_dir):
     def _paint_mask(mask):
         # TODO: check if mask colors are random
         return np.array([[0, 0, 0, 0], [255, 203, 0, 153]],
-            dtype=np.uint8)[mask.astype(np.uint8)]
+            dtype=np.uint8)[mask.astype(np.uint8)]

datumaro/plugins/mots_format.py

@@ -7,9 +7,9 @@
 from enum import Enum
 from glob import iglob
 import logging as log
-import numpy as np
 import os
 import os.path as osp
+import numpy as np
 
 from datumaro.components.extractor import (SourceExtractor, Importer,
     DatasetItem, AnnotationType, Mask, LabelCategories

datumaro/plugins/openvino_plugin/launcher.py

@@ -5,11 +5,11 @@
 
 # pylint: disable=exec-used
 
-import cv2
-import logging as log
-import numpy as np
 import os.path as osp
 import shutil
+import logging as log
+import cv2
+import numpy as np
 
 from openvino.inference_engine import IECore
 

datumaro/plugins/openvino_plugin/samples/mobilenet_v2_pytorch_interp.py

@@ -14,7 +14,7 @@ def process_outputs(inputs, outputs):
     # results = conversion result; [[ Annotation, ... ], ... ]
 
     results = []
-    for input, output in zip(inputs, outputs):
+    for input_, output in zip(inputs, outputs): # pylint: disable=unused-variable
         image_results = []
         output = softmax(output).tolist()
         label = output.index(max(output))

datumaro/plugins/openvino_plugin/samples/ssd_face_detection_interp.py

@@ -41,9 +41,9 @@ def process_outputs(inputs, outputs):
     # results = conversion result; [[ Annotation, ... ], ... ]
 
     results = []
-    for input, detections in zip(inputs, outputs["detection_out"]):
+    for input_, detections in zip(inputs, outputs["detection_out"]):
 
-        input_height, input_width = input.shape[:2]
+        input_height, input_width = input_.shape[:2]
 
         confs = outputs["Softmax_189/Softmax_"]
         detections = detections[0]
@@ -52,7 +52,7 @@ def process_outputs(inputs, outputs):
 
         image_results = []
         for i, det in enumerate(detections):
-            image_id = int(det[0])
+            image_id = int(det[0]) # pylint: disable=unused-variable
             label = int(det[1])
             conf = float(det[2])
             det_confs = confs[conf_ids[i]]

datumaro/plugins/openvino_plugin/samples/ssd_mobilenet_coco_detection_interp.py

@@ -42,11 +42,11 @@ def process_outputs(inputs, outputs):
     # results = conversion result; [[ Annotation, ... ], ... ]
 
     results = []
-    for input, confs, detections in zip(
+    for input_, confs, detections in zip(
         inputs, outputs["do_ExpandDims_conf/sigmoid"], outputs["DetectionOutput"]
     ):
 
-        input_height, input_width = input.shape[:2]
+        input_height, input_width = input_.shape[:2]
 
         confs = confs[0].reshape(-1, model_class_num)
         detections = detections[0]
@@ -55,7 +55,7 @@ def process_outputs(inputs, outputs):
 
         image_results = []
         for i, det in enumerate(detections):
-            image_id = int(det[0])
+            image_id = int(det[0]) # pylint: disable=unused-variable
             label = int(det[1])
             conf = float(det[2])
             det_confs = confs[conf_ids[i]]

datumaro/plugins/openvino_plugin/samples/ssd_person_detection_interp.py

@@ -41,9 +41,9 @@ def process_outputs(inputs, outputs):
     # results = conversion result; [[ Annotation, ... ], ... ]
 
     results = []
-    for input, detections in zip(inputs, outputs["detection_out"]):
+    for input_, detections in zip(inputs, outputs["detection_out"]):
 
-        input_height, input_width = input.shape[:2]
+        input_height, input_width = input_.shape[:2]
 
         confs = outputs["Softmax_189/Softmax_"]
         detections = detections[0]
@@ -52,7 +52,7 @@ def process_outputs(inputs, outputs):
 
         image_results = []
         for i, det in enumerate(detections):
-            image_id = int(det[0])
+            image_id = int(det[0]) # pylint: disable=unused-variable
             label = int(det[1])
             conf = float(det[2])
             det_confs = confs[conf_ids[i]]

datumaro/plugins/openvino_plugin/samples/ssd_person_vehicle_bike_detection_interp.py

@@ -41,9 +41,9 @@ def process_outputs(inputs, outputs):
     # results = conversion result; [[ Annotation, ... ], ... ]
 
     results = []
-    for input, detections in zip(inputs, outputs["detection_out"]):
+    for input_, detections in zip(inputs, outputs["detection_out"]):
 
-        input_height, input_width = input.shape[:2]
+        input_height, input_width = input_.shape[:2]
 
         confs = outputs["Softmax_189/Softmax_"]
         detections = detections[0]
@@ -52,7 +52,7 @@ def process_outputs(inputs, outputs):
 
         image_results = []
         for i, det in enumerate(detections):
-            image_id = int(det[0])
+            image_id = int(det[0]) # pylint: disable=unused-variable
             label = int(det[1])
             conf = float(det[2])
             det_confs = confs[conf_ids[i]]

datumaro/plugins/openvino_plugin/samples/ssd_vehicle_detection_interp.py

@@ -41,9 +41,9 @@ def process_outputs(inputs, outputs):
     # results = conversion result; [[ Annotation, ... ], ... ]
 
     results = []
-    for input, detections in zip(inputs, outputs["detection_out"]):
+    for input_, detections in zip(inputs, outputs["detection_out"]):
 
-        input_height, input_width = input.shape[:2]
+        input_height, input_width = input_.shape[:2]
 
         confs = outputs["Softmax_189/Softmax_"]
         detections = detections[0]
@@ -52,7 +52,7 @@ def process_outputs(inputs, outputs):
 
         image_results = []
         for i, det in enumerate(detections):
-            image_id = int(det[0])
+            image_id = int(det[0]) # pylint: disable=unused-variable
             label = int(det[1])
             conf = float(det[2])
             det_confs = confs[conf_ids[i]]

datumaro/plugins/splitter.py

@@ -3,10 +3,10 @@
 # SPDX-License-Identifier: MIT
 
 import logging as log
-import numpy as np
 import copy
 from math import gcd
 from enum import Enum, auto
+import numpy as np
 
 from datumaro.components.extractor import (Transform, AnnotationType,
     DEFAULT_SUBSET_NAME)

datumaro/plugins/tf_detection_api_format/converter.py

@@ -11,9 +11,7 @@
 import os.path as osp
 import string
 
-from datumaro.components.extractor import (AnnotationType, DEFAULT_SUBSET_NAME,
-    LabelCategories
-)
+from datumaro.components.extractor import (AnnotationType, LabelCategories)
 from datumaro.components.converter import Converter
 from datumaro.util.image import encode_image, ByteImage
 from datumaro.util.annotation_util import (max_bbox,
@@ -196,7 +194,7 @@ def _make_tf_example(self, item):
 
         return tf_example
 
-    def _save_image(self, item, path=None):
+    def _save_image(self, item, path=None): # pylint: disable=arguments-differ
         src_ext = item.image.ext.lower()
         dst_ext = osp.splitext(osp.basename(path))[1].lower()
         fmt = DetectionApiPath.IMAGE_EXT_FORMAT.get(dst_ext, '')