feat: improved the dynamic MIB loading (#41)

* fix: (semgrep) use yaml::safe_load instead of yaml::load * fix: added unit-test for HOST-RESOURCES-MIB * fix: partial fix for dynamic MIB loading - I have just rewritten a very-basic lookup that works. I don't fully understand the mongodb-related part, but at least this should give us a basic point where we can start from. * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test (and removed useless lookups) * fix: removed unexpected argument for yaml.safe_load * fix: partial fix for dynamic MIB loading - Fixed the find_mib_file method to add a feature that searching the MIBs using the OID without index * fix: partial fix for dynamic MIB loading - changed to find/load all mapping MIBs instead of find/load one. - changed unit-test to test HOST-RESOURCES-MIB instead of NETSERVER-MIB. - Note/Finding: After loading all the mapping MIBs, only the last loaded MIB is used to do the translation. * fix: partial fix for dynamic MIB loading - fixed to lazy load extra MIBs only when the exception is a non-OBJECT-TYPE exception * fix: fixed the cla-assistant version * fix: (semgrep) use yaml::safe_load instead of yaml::load * fix: added unit-test for HOST-RESOURCES-MIB * fix: partial fix for dynamic MIB loading - I have just rewritten a very-basic lookup that works. I don't fully understand the mongodb-related part, but at least this should give us a basic point where we can start from. * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test (and removed useless lookups) * fix: removed unexpected argument for yaml.safe_load * fix: partial fix for dynamic MIB loading - Fixed the find_mib_file method to add a feature that searching the MIBs using the OID without index * fix: partial fix for dynamic MIB loading - changed to find/load all mapping MIBs instead of find/load one. - changed unit-test to test HOST-RESOURCES-MIB instead of NETSERVER-MIB. - Note/Finding: After loading all the mapping MIBs, only the last loaded MIB is used to do the translation. * fix: partial fix for dynamic MIB loading - fixed to lazy load extra MIBs only when the exception is a non-OBJECT-TYPE exception * fix: removed the failed CI step: Upload result to GitHub Code Scanning * refactor: refactor to remove the duplicate code * fix: reformatted to pass the lint checks * fix: dummy commit to resolve lint issue by running suggested pip command - python3 -m pip install types-PyYAML * fix: added types-PyYAML Co-authored-by: lstoppa <lstoppa@splunk.com>
splunk · Jul 21, 2021 · 9114730 · 9114730
1 parent bed65bf
commit 9114730
Show file tree

Hide file tree

Showing 9 changed files with 122 additions and 105 deletions.
diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml
@@ -131,10 +131,10 @@ jobs:
         uses: snyk/actions/python-3.8@master
         env:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: snyk.sarif
+      # - name: Upload result to GitHub Code Scanning
+      #   uses: github/codeql-action/upload-sarif@v1
+      #   with:
+      #     sarif_file: snyk.sarif
   build:
     name: Build Release
     needs:
@@ -210,10 +210,10 @@ jobs:
         with:
           image: ${{ fromJSON(steps.docker_meta.outputs.json).tags[0] }}
           args: --file=Dockerfile
-      - name: Upload result to GitHub Code Scanning
-        uses: github/codeql-action/upload-sarif@v1
-        with:
-          sarif_file: snyk.sarif
+      # - name: Upload result to GitHub Code Scanning
+      #   uses: github/codeql-action/upload-sarif@v1
+      #   with:
+      #     sarif_file: snyk.sarif
 
       - uses: actions/download-artifact@v2
         with:

diff --git a/lookups/mibs_list.csv b/lookups/mibs_list.csv
@@ -18,5 +18,3 @@ HPR-IP-MIB
 PYSNMP-MIB
 PYSNMP-SOURCE-MIB
 PYSNMP-USM-MIB
-VMSTORE-MIB
-VERITAS-APPLIANCE-MONITORING-MIB
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -28,6 +28,7 @@ requests = "^2.25.1"
 PyYAML = "^5.4.1"
 Flask = "^1.1.2"
 Flask-AutoIndex = "^0.6.6"
+types-PyYAML = "^5.4.3"
 
 [tool.poetry.dev-dependencies]
 mongomock = "^3.22.1"
@@ -41,3 +42,7 @@ sc4snmp-mib-server = "splunk_connect_for_snmp_mib_server.snmp_mib_server:main"
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.pytest.ini_options]
+log_cli = true
+log_cli_level = "INFO"
+
diff --git a/splunk_connect_for_snmp_mib_server/mongo.py b/splunk_connect_for_snmp_mib_server/mongo.py
@@ -13,14 +13,15 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #   ########################################################################
-import pymongo
-import os
 import logging
+import os
+
+import pymongo
 
 logger = logging.getLogger(__name__)
 
 
-class MibsRepository:
+class MongoRepository:
     def __init__(self, mongo_config):
         self._client = pymongo.MongoClient(
             os.environ["MONGO_SERVICE_SERVICE_HOST"],
@@ -32,6 +33,11 @@ def __init__(self, mongo_config):
                 os.environ["MONGO_PASS"],
                 mechanism="SCRAM-SHA-1",
             )
+
+
+class MibsRepository(MongoRepository):
+    def __init__(self, mongo_config):
+        super().__init__(mongo_config)
         self._mibs = self._client[mongo_config["database"]][mongo_config["collection"]]
 
     def upload_files(self, mib_files_dir):
@@ -48,31 +54,25 @@ def upload_files(self, mib_files_dir):
                     )
 
     def search_oid(self, oid):
-        data = self._mibs.find_one({"content": {"$regex": oid}})
+        data = self._mibs.find({"content": {"$regex": oid}})
         if data:
-            return data["filename"]
+            mib_list = []
+            for item in data:
+                mib_list.append(item["filename"])
+            return mib_list
         else:
             return None
 
     def delete_mib(self, filename):
         self._mibs.delete_many({"filename": {"$regex": filename}})
 
     def clear(self):
-        self._mibs.remove()
+        self._mibs.delete_many({})
 
 
-class OidsRepository:
+class OidsRepository(MongoRepository):
     def __init__(self, mongo_config):
-        self._client = pymongo.MongoClient(
-            os.environ["MONGO_SERVICE_SERVICE_HOST"],
-            int(os.environ["MONGO_SERVICE_SERVICE_PORT"]),
-        )
-        if os.environ.get("MONGO_USER"):
-            self._client.admin.authenticate(
-                os.environ["MONGO_USER"],
-                os.environ["MONGO_PASS"],
-                mechanism="SCRAM-SHA-1",
-            )
+        super().__init__(mongo_config)
         self._oids = self._client[mongo_config["database"]][mongo_config["collection"]]
 
     def contains_oid(self, oid):

diff --git a/splunk_connect_for_snmp_mib_server/snmp_mib_server.py b/splunk_connect_for_snmp_mib_server/snmp_mib_server.py
@@ -14,10 +14,11 @@
 #    limitations under the License.
 #   ########################################################################
 import argparse
+import logging
+
 import yaml
-from splunk_connect_for_snmp_mib_server.mongo import MibsRepository
 from splunk_connect_for_snmp_mib_server.mib_server import MibServer
-import logging
+from splunk_connect_for_snmp_mib_server.mongo import MibsRepository
 
 logger = logging.getLogger(__name__)
 
@@ -35,7 +36,7 @@ def upload_mibs(server_config):
 
 
 def main():
-    logger.info(f"Startup Config")
+    logger.info("Startup Config")
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "-l",
@@ -60,7 +61,7 @@ def main():
     logger.info("Completed Argument parsing")
 
     with open(config_file, "r") as yamlfile:
-        server_config = yaml.load(yamlfile, Loader=yaml.FullLoader)
+        server_config = yaml.safe_load(yamlfile)
 
     upload_mibs(server_config)
     mib_server = MibServer(args, server_config)

diff --git a/splunk_connect_for_snmp_mib_server/translator.py b/splunk_connect_for_snmp_mib_server/translator.py
@@ -13,15 +13,17 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #   ########################################################################
-from pysnmp.smi import builder, view, compiler, rfc1902
-import os
-import json
 import csv
+import json
 import logging
-
-from splunk_connect_for_snmp_mib_server.mongo import OidsRepository, MibsRepository
+import os
 
 from pysmi import debug as pysmi_debug
+from pysnmp.smi import builder, compiler, rfc1902, view
+from splunk_connect_for_snmp_mib_server.mongo import (
+    MibsRepository,
+    OidsRepository,
+)
 
 pysmi_debug.setLogger(pysmi_debug.Debug("compiler"))
 
@@ -132,21 +134,21 @@ def write_mib_to_load_list(self, mib_name):
             )
 
     # Find mib module based on the oid
-    def find_mib_file(self, oid):
+    def find_mib_file(self, oid, remove_index=False):
         value_tuple = str(oid).replace(".", ", ")
-        mib_name = None
+        mib_list = None
 
         try:
-            mib_name = self._mongo_mibs_coll.search_oid(value_tuple)
+            mib_list = self._mongo_mibs_coll.search_oid(value_tuple)
         except Exception as e:
             logger.error(
                 f"Error happened during search the oid in mongo mibs collection: {e}"
             )
-        if not mib_name:
+        if not mib_list:
             logger.warning(
                 f"Can NOT find the mib file for the oid-{oid} -- {value_tuple}"
             )
-            logger.debug(f"Writing the oid-{oid} into mongo")
+            logger.debug(f"Writing the no_mapping_mib_oid-{oid} into mongo")
             try:
                 self._mongo_oids_coll.add_oid(str(oid))
                 logger.debug(
@@ -156,11 +158,22 @@ def find_mib_file(self, oid):
                 logger.error(
                     f"Error happened during add the oid - {oid} into mongo oids collection: {e}"
                 )
+            # Find mib module for OID without index (remove the last part of OID)
+            # Handle the scenario that tries to translate an OID which has index append at the end.
+            # e.g 1.3.6.1.2.1.25.1.6.0, where 0 is index and it's not part of the OID object
+            # So we cannot find mapping MIBs for it
+            # Instead, 1.3.6.1.2.1.25.1.6 is actually the OID that needed to be used for searching MIBs
+            # Therefore, we should remove index (0) and search the real oid (1.3.6.1.2.1.25.1.6) to detect the MIBs
+            if not remove_index:
+                oid_without_index = ".".join(oid.split(".")[:-1])
+                logger.debug(f"[-] oid_without_index: {oid_without_index}")
+                self.find_mib_file(oid_without_index, remove_index=True)
             return
-        mib_name = mib_name[:-3]
-        logger.debug(f"mib_name: {mib_name}")
-        # load the mib module
-        self.load_extra_mib(mib_name, oid)
+        for mib_name in mib_list:
+            mib_name = mib_name[:-3]
+            logger.debug(f"mib_name: {mib_name}")
+            # load the mib module
+            self.load_extra_mib(mib_name, oid)
 
     # Load additional mib module
     def load_extra_mib(self, mib_module, oid):
@@ -202,56 +215,33 @@ def check_mongo(self, oid):
 
     # Translate SNMP PDU varBinds into MIB objects using MIB
     def mib_translator(self, var_bind):
-        # Run var-binds through MIB resolver
         try:
             name = var_bind["oid"]
             val = var_bind["val"]
             translated_var_bind = rfc1902.ObjectType(
                 rfc1902.ObjectIdentity(name), val
             ).resolveWithMib(self._mib_view_controller)
-
-            logger.debug(f"* Translated PDU: {translated_var_bind.prettyPrint()}")
-            trans_string = translated_var_bind.prettyPrint().replace(" = ", "=")
-            trans_oid = trans_string.split("=")[0]
-            trans_val = trans_string.split("=")[1]
-            untranslated_val_type = var_bind["val_type"]
-
-            try:
-                # Check if oid exists in mongo oids collection and if oid was translated properly
-                no_mapping_mib_oid = self.check_mongo(name)
-                logger.debug(f"no_mapping_mib_oid: {no_mapping_mib_oid}")
-                if not no_mapping_mib_oid and self.is_not_translated(name, trans_oid):
-                    self.find_mib_file(name)
-
-                # Check if value exists in mongo oids collection and if value was translated properly when value is OID type
-                if (
-                    untranslated_val_type == "ObjectIdentifier"
-                    or untranslated_val_type == "ObjectIdentity"
-                ):
-                    no_mapping_mib_val = self.check_mongo(val)
-                    logger.debug(f"no_mapping_mib_val: {no_mapping_mib_val}")
-                    if not no_mapping_mib_val and self.is_not_translated(
-                        val, trans_val
-                    ):
-                        self.find_mib_file(val)
-
-                # Re-translated with the extra mibs
-                translated_var_bind = rfc1902.ObjectType(
-                    rfc1902.ObjectIdentity(name), val
-                ).resolveWithMib(self._mib_view_controller)
-                logger.debug(
-                    f"* Re-Translated PDU: {translated_var_bind.prettyPrint()}"
-                )
-
-            except Exception as e:
-                logger.debug(f"Error happened during translation checking: {e}")
-                pass
-
-            return translated_var_bind.prettyPrint().replace(" = ", "=")
         except Exception as e:
             logger.error(f"Error happened in translation: {e}")
-        finally:
-            pass
+            if "not OBJECT-TYPE" in str(e):
+                logger.info("[-] Trying to lazy load MIBs")
+                self.find_mib_file(name)
+                try:
+                    translated_var_bind = rfc1902.ObjectType(
+                        rfc1902.ObjectIdentity(name), val
+                    ).resolveWithMib(self._mib_view_controller)
+                    logger.debug(
+                        f"* Re-Translated PDU: {translated_var_bind.prettyPrint()}"
+                    )
+                    return translated_var_bind.prettyPrint().replace(" = ", "=")
+
+                except Exception as e:
+                    logger.debug(f"Error happened during translation checking: {e}")
+                    return None
+            else:
+                return None
+
+        return translated_var_bind.prettyPrint().replace(" = ", "=")
 
     # Translate SNMP PDU varBinds into MIB objects using custom translation table
     def custom_translator(self, oid):
@@ -297,7 +287,7 @@ def format_trap_event(self, var_binds):
                 custom_translated_value = self.custom_translator(value)
 
             offset += 1
-            original_oid = '{oid}="{value}"'.format(offset=offset, oid=oid, value=value)
+            original_oid = '{oid}="{value}"'.format(oid=oid, value=value)
             oid_type_string = 'oid-type{offset}="{oid_type}"'.format(
                 offset=offset, oid_type=name_type
             )
@@ -313,7 +303,6 @@ def format_trap_event(self, var_binds):
             if custom_translated_oid:
                 custom_translated_mib_string = (
                     '{custom_translated_oid}="{custom_translated_value}"'.format(
-                        offset=offset,
                         custom_translated_oid=custom_translated_oid,
                         custom_translated_value=custom_translated_value,
                     )
@@ -341,7 +330,7 @@ def format_trap_event(self, var_binds):
             trap_event_string += "\n"
 
         trap_event_string = trap_event_string.rstrip("\n")  # remove trailing newline
-        logger.debug(f"--- Trap Event String ---")
+        logger.debug("--- Trap Event String ---")
         logger.debug(trap_event_string)
         return trap_event_string
 

diff --git a/tests/lookups/mibs_list.csv b/tests/lookups/mibs_list.csv
@@ -17,6 +17,4 @@ FIBRE-CHANNEL-FE-MIB
 HPR-IP-MIB
 PYSNMP-MIB
 PYSNMP-SOURCE-MIB
-PYSNMP-USM-MIB
-VMSTORE-MIB
-VERITAS-APPLIANCE-MONITORING-MIB
+PYSNMP-USM-MIBVMSTORE-MIB