From 9114730c13ade794ab35513e5579f2a59f17defb Mon Sep 17 00:00:00 2001 From: Yuan Ling <32370701+lingy1028@users.noreply.github.com> Date: Wed, 21 Jul 2021 00:30:47 -0700 Subject: [PATCH] feat: improved the dynamic MIB loading (#41) * fix: (semgrep) use yaml::safe_load instead of yaml::load * fix: added unit-test for HOST-RESOURCES-MIB * fix: partial fix for dynamic MIB loading - I have just rewritten a very-basic lookup that works. I don't fully understand the mongodb-related part, but at least this should give us a basic point where we can start from. * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test (and removed useless lookups) * fix: removed unexpected argument for yaml.safe_load * fix: partial fix for dynamic MIB loading - Fixed the find_mib_file method to add a feature that searching the MIBs using the OID without index * fix: partial fix for dynamic MIB loading - changed to find/load all mapping MIBs instead of find/load one. - changed unit-test to test HOST-RESOURCES-MIB instead of NETSERVER-MIB. - Note/Finding: After loading all the mapping MIBs, only the last loaded MIB is used to do the translation. * fix: partial fix for dynamic MIB loading - fixed to lazy load extra MIBs only when the exception is a non-OBJECT-TYPE exception * fix: fixed the cla-assistant version * fix: (semgrep) use yaml::safe_load instead of yaml::load * fix: added unit-test for HOST-RESOURCES-MIB * fix: partial fix for dynamic MIB loading - I have just rewritten a very-basic lookup that works. I don't fully understand the mongodb-related part, but at least this should give us a basic point where we can start from. * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test * fix: partial fix for dynamic MIB loading - Fixed one MIB file name in unit-test (and removed useless lookups) * fix: removed unexpected argument for yaml.safe_load * fix: partial fix for dynamic MIB loading - Fixed the find_mib_file method to add a feature that searching the MIBs using the OID without index * fix: partial fix for dynamic MIB loading - changed to find/load all mapping MIBs instead of find/load one. - changed unit-test to test HOST-RESOURCES-MIB instead of NETSERVER-MIB. - Note/Finding: After loading all the mapping MIBs, only the last loaded MIB is used to do the translation. * fix: partial fix for dynamic MIB loading - fixed to lazy load extra MIBs only when the exception is a non-OBJECT-TYPE exception * fix: removed the failed CI step: Upload result to GitHub Code Scanning * refactor: refactor to remove the duplicate code * fix: reformatted to pass the lint checks * fix: dummy commit to resolve lint issue by running suggested pip command - python3 -m pip install types-PyYAML * fix: added types-PyYAML Co-authored-by: lstoppa --- .github/workflows/build-test-release.yml | 16 +-- lookups/mibs_list.csv | 2 - poetry.lock | 16 ++- pyproject.toml | 5 + splunk_connect_for_snmp_mib_server/mongo.py | 34 +++--- .../snmp_mib_server.py | 9 +- .../translator.py | 107 ++++++++---------- tests/lookups/mibs_list.csv | 4 +- tests/test_translator.py | 34 ++++-- 9 files changed, 122 insertions(+), 105 deletions(-) diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml index 30b9ca07..cfe8e5e2 100644 --- a/.github/workflows/build-test-release.yml +++ b/.github/workflows/build-test-release.yml @@ -131,10 +131,10 @@ jobs: uses: snyk/actions/python-3.8@master env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - - name: Upload result to GitHub Code Scanning - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: snyk.sarif + # - name: Upload result to GitHub Code Scanning + # uses: github/codeql-action/upload-sarif@v1 + # with: + # sarif_file: snyk.sarif build: name: Build Release needs: @@ -210,10 +210,10 @@ jobs: with: image: ${{ fromJSON(steps.docker_meta.outputs.json).tags[0] }} args: --file=Dockerfile - - name: Upload result to GitHub Code Scanning - uses: github/codeql-action/upload-sarif@v1 - with: - sarif_file: snyk.sarif + # - name: Upload result to GitHub Code Scanning + # uses: github/codeql-action/upload-sarif@v1 + # with: + # sarif_file: snyk.sarif - uses: actions/download-artifact@v2 with: diff --git a/lookups/mibs_list.csv b/lookups/mibs_list.csv index a20958a7..c1f2389f 100644 --- a/lookups/mibs_list.csv +++ b/lookups/mibs_list.csv @@ -18,5 +18,3 @@ HPR-IP-MIB PYSNMP-MIB PYSNMP-SOURCE-MIB PYSNMP-USM-MIB -VMSTORE-MIB -VERITAS-APPLIANCE-MONITORING-MIB \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 542402dc..2c03145e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -363,6 +363,14 @@ category = "dev" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +[[package]] +name = "types-pyyaml" +version = "5.4.3" +description = "Typing stubs for PyYAML" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "urllib3" version = "1.26.5" @@ -391,7 +399,7 @@ watchdog = ["watchdog"] [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "81fadf1167f12b722ee117aad87724fff433e514859e8b63d45e78faa8e500b4" +content-hash = "6b475cd89e807a601526e833df3a99b6b192d3e95f8f7eb9f2848154070a5021" [metadata.files] atomicwrites = [ @@ -416,6 +424,7 @@ click = [ ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, @@ -489,6 +498,7 @@ idna = [ {file = "idna-2.10.tar.gz", hash = "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6"}, ] iniconfig = [ + {file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"}, {file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"}, ] itsdangerous = [ @@ -748,6 +758,10 @@ toml = [ {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +types-pyyaml = [ + {file = "types-PyYAML-5.4.3.tar.gz", hash = "sha256:2e7b81b2b7af751634425107b986086c6ba7cb61270a43a5c290c58be8cdbc3a"}, + {file = "types_PyYAML-5.4.3-py2.py3-none-any.whl", hash = "sha256:bca83cbfc0be48600a8abf1e3d87fb762a91e6d35d724029a3321dd2dce2ceb1"}, +] urllib3 = [ {file = "urllib3-1.26.5-py2.py3-none-any.whl", hash = "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c"}, {file = "urllib3-1.26.5.tar.gz", hash = "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"}, diff --git a/pyproject.toml b/pyproject.toml index cadbea71..76fc7739 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ requests = "^2.25.1" PyYAML = "^5.4.1" Flask = "^1.1.2" Flask-AutoIndex = "^0.6.6" +types-PyYAML = "^5.4.3" [tool.poetry.dev-dependencies] mongomock = "^3.22.1" @@ -41,3 +42,7 @@ sc4snmp-mib-server = "splunk_connect_for_snmp_mib_server.snmp_mib_server:main" requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" +[tool.pytest.ini_options] +log_cli = true +log_cli_level = "INFO" + diff --git a/splunk_connect_for_snmp_mib_server/mongo.py b/splunk_connect_for_snmp_mib_server/mongo.py index eb8240e4..4232fe14 100644 --- a/splunk_connect_for_snmp_mib_server/mongo.py +++ b/splunk_connect_for_snmp_mib_server/mongo.py @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # ######################################################################## -import pymongo -import os import logging +import os + +import pymongo logger = logging.getLogger(__name__) -class MibsRepository: +class MongoRepository: def __init__(self, mongo_config): self._client = pymongo.MongoClient( os.environ["MONGO_SERVICE_SERVICE_HOST"], @@ -32,6 +33,11 @@ def __init__(self, mongo_config): os.environ["MONGO_PASS"], mechanism="SCRAM-SHA-1", ) + + +class MibsRepository(MongoRepository): + def __init__(self, mongo_config): + super().__init__(mongo_config) self._mibs = self._client[mongo_config["database"]][mongo_config["collection"]] def upload_files(self, mib_files_dir): @@ -48,9 +54,12 @@ def upload_files(self, mib_files_dir): ) def search_oid(self, oid): - data = self._mibs.find_one({"content": {"$regex": oid}}) + data = self._mibs.find({"content": {"$regex": oid}}) if data: - return data["filename"] + mib_list = [] + for item in data: + mib_list.append(item["filename"]) + return mib_list else: return None @@ -58,21 +67,12 @@ def delete_mib(self, filename): self._mibs.delete_many({"filename": {"$regex": filename}}) def clear(self): - self._mibs.remove() + self._mibs.delete_many({}) -class OidsRepository: +class OidsRepository(MongoRepository): def __init__(self, mongo_config): - self._client = pymongo.MongoClient( - os.environ["MONGO_SERVICE_SERVICE_HOST"], - int(os.environ["MONGO_SERVICE_SERVICE_PORT"]), - ) - if os.environ.get("MONGO_USER"): - self._client.admin.authenticate( - os.environ["MONGO_USER"], - os.environ["MONGO_PASS"], - mechanism="SCRAM-SHA-1", - ) + super().__init__(mongo_config) self._oids = self._client[mongo_config["database"]][mongo_config["collection"]] def contains_oid(self, oid): diff --git a/splunk_connect_for_snmp_mib_server/snmp_mib_server.py b/splunk_connect_for_snmp_mib_server/snmp_mib_server.py index 6bea7d07..e14ccb0a 100644 --- a/splunk_connect_for_snmp_mib_server/snmp_mib_server.py +++ b/splunk_connect_for_snmp_mib_server/snmp_mib_server.py @@ -14,10 +14,11 @@ # limitations under the License. # ######################################################################## import argparse +import logging + import yaml -from splunk_connect_for_snmp_mib_server.mongo import MibsRepository from splunk_connect_for_snmp_mib_server.mib_server import MibServer -import logging +from splunk_connect_for_snmp_mib_server.mongo import MibsRepository logger = logging.getLogger(__name__) @@ -35,7 +36,7 @@ def upload_mibs(server_config): def main(): - logger.info(f"Startup Config") + logger.info("Startup Config") parser = argparse.ArgumentParser() parser.add_argument( "-l", @@ -60,7 +61,7 @@ def main(): logger.info("Completed Argument parsing") with open(config_file, "r") as yamlfile: - server_config = yaml.load(yamlfile, Loader=yaml.FullLoader) + server_config = yaml.safe_load(yamlfile) upload_mibs(server_config) mib_server = MibServer(args, server_config) diff --git a/splunk_connect_for_snmp_mib_server/translator.py b/splunk_connect_for_snmp_mib_server/translator.py index 7baf8cb2..d9830326 100644 --- a/splunk_connect_for_snmp_mib_server/translator.py +++ b/splunk_connect_for_snmp_mib_server/translator.py @@ -13,15 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # ######################################################################## -from pysnmp.smi import builder, view, compiler, rfc1902 -import os -import json import csv +import json import logging - -from splunk_connect_for_snmp_mib_server.mongo import OidsRepository, MibsRepository +import os from pysmi import debug as pysmi_debug +from pysnmp.smi import builder, compiler, rfc1902, view +from splunk_connect_for_snmp_mib_server.mongo import ( + MibsRepository, + OidsRepository, +) pysmi_debug.setLogger(pysmi_debug.Debug("compiler")) @@ -132,21 +134,21 @@ def write_mib_to_load_list(self, mib_name): ) # Find mib module based on the oid - def find_mib_file(self, oid): + def find_mib_file(self, oid, remove_index=False): value_tuple = str(oid).replace(".", ", ") - mib_name = None + mib_list = None try: - mib_name = self._mongo_mibs_coll.search_oid(value_tuple) + mib_list = self._mongo_mibs_coll.search_oid(value_tuple) except Exception as e: logger.error( f"Error happened during search the oid in mongo mibs collection: {e}" ) - if not mib_name: + if not mib_list: logger.warning( f"Can NOT find the mib file for the oid-{oid} -- {value_tuple}" ) - logger.debug(f"Writing the oid-{oid} into mongo") + logger.debug(f"Writing the no_mapping_mib_oid-{oid} into mongo") try: self._mongo_oids_coll.add_oid(str(oid)) logger.debug( @@ -156,11 +158,22 @@ def find_mib_file(self, oid): logger.error( f"Error happened during add the oid - {oid} into mongo oids collection: {e}" ) + # Find mib module for OID without index (remove the last part of OID) + # Handle the scenario that tries to translate an OID which has index append at the end. + # e.g 1.3.6.1.2.1.25.1.6.0, where 0 is index and it's not part of the OID object + # So we cannot find mapping MIBs for it + # Instead, 1.3.6.1.2.1.25.1.6 is actually the OID that needed to be used for searching MIBs + # Therefore, we should remove index (0) and search the real oid (1.3.6.1.2.1.25.1.6) to detect the MIBs + if not remove_index: + oid_without_index = ".".join(oid.split(".")[:-1]) + logger.debug(f"[-] oid_without_index: {oid_without_index}") + self.find_mib_file(oid_without_index, remove_index=True) return - mib_name = mib_name[:-3] - logger.debug(f"mib_name: {mib_name}") - # load the mib module - self.load_extra_mib(mib_name, oid) + for mib_name in mib_list: + mib_name = mib_name[:-3] + logger.debug(f"mib_name: {mib_name}") + # load the mib module + self.load_extra_mib(mib_name, oid) # Load additional mib module def load_extra_mib(self, mib_module, oid): @@ -202,56 +215,33 @@ def check_mongo(self, oid): # Translate SNMP PDU varBinds into MIB objects using MIB def mib_translator(self, var_bind): - # Run var-binds through MIB resolver try: name = var_bind["oid"] val = var_bind["val"] translated_var_bind = rfc1902.ObjectType( rfc1902.ObjectIdentity(name), val ).resolveWithMib(self._mib_view_controller) - - logger.debug(f"* Translated PDU: {translated_var_bind.prettyPrint()}") - trans_string = translated_var_bind.prettyPrint().replace(" = ", "=") - trans_oid = trans_string.split("=")[0] - trans_val = trans_string.split("=")[1] - untranslated_val_type = var_bind["val_type"] - - try: - # Check if oid exists in mongo oids collection and if oid was translated properly - no_mapping_mib_oid = self.check_mongo(name) - logger.debug(f"no_mapping_mib_oid: {no_mapping_mib_oid}") - if not no_mapping_mib_oid and self.is_not_translated(name, trans_oid): - self.find_mib_file(name) - - # Check if value exists in mongo oids collection and if value was translated properly when value is OID type - if ( - untranslated_val_type == "ObjectIdentifier" - or untranslated_val_type == "ObjectIdentity" - ): - no_mapping_mib_val = self.check_mongo(val) - logger.debug(f"no_mapping_mib_val: {no_mapping_mib_val}") - if not no_mapping_mib_val and self.is_not_translated( - val, trans_val - ): - self.find_mib_file(val) - - # Re-translated with the extra mibs - translated_var_bind = rfc1902.ObjectType( - rfc1902.ObjectIdentity(name), val - ).resolveWithMib(self._mib_view_controller) - logger.debug( - f"* Re-Translated PDU: {translated_var_bind.prettyPrint()}" - ) - - except Exception as e: - logger.debug(f"Error happened during translation checking: {e}") - pass - - return translated_var_bind.prettyPrint().replace(" = ", "=") except Exception as e: logger.error(f"Error happened in translation: {e}") - finally: - pass + if "not OBJECT-TYPE" in str(e): + logger.info("[-] Trying to lazy load MIBs") + self.find_mib_file(name) + try: + translated_var_bind = rfc1902.ObjectType( + rfc1902.ObjectIdentity(name), val + ).resolveWithMib(self._mib_view_controller) + logger.debug( + f"* Re-Translated PDU: {translated_var_bind.prettyPrint()}" + ) + return translated_var_bind.prettyPrint().replace(" = ", "=") + + except Exception as e: + logger.debug(f"Error happened during translation checking: {e}") + return None + else: + return None + + return translated_var_bind.prettyPrint().replace(" = ", "=") # Translate SNMP PDU varBinds into MIB objects using custom translation table def custom_translator(self, oid): @@ -297,7 +287,7 @@ def format_trap_event(self, var_binds): custom_translated_value = self.custom_translator(value) offset += 1 - original_oid = '{oid}="{value}"'.format(offset=offset, oid=oid, value=value) + original_oid = '{oid}="{value}"'.format(oid=oid, value=value) oid_type_string = 'oid-type{offset}="{oid_type}"'.format( offset=offset, oid_type=name_type ) @@ -313,7 +303,6 @@ def format_trap_event(self, var_binds): if custom_translated_oid: custom_translated_mib_string = ( '{custom_translated_oid}="{custom_translated_value}"'.format( - offset=offset, custom_translated_oid=custom_translated_oid, custom_translated_value=custom_translated_value, ) @@ -341,7 +330,7 @@ def format_trap_event(self, var_binds): trap_event_string += "\n" trap_event_string = trap_event_string.rstrip("\n") # remove trailing newline - logger.debug(f"--- Trap Event String ---") + logger.debug("--- Trap Event String ---") logger.debug(trap_event_string) return trap_event_string diff --git a/tests/lookups/mibs_list.csv b/tests/lookups/mibs_list.csv index a20958a7..dc55dd8f 100644 --- a/tests/lookups/mibs_list.csv +++ b/tests/lookups/mibs_list.csv @@ -17,6 +17,4 @@ FIBRE-CHANNEL-FE-MIB HPR-IP-MIB PYSNMP-MIB PYSNMP-SOURCE-MIB -PYSNMP-USM-MIB -VMSTORE-MIB -VERITAS-APPLIANCE-MONITORING-MIB \ No newline at end of file +PYSNMP-USM-MIBVMSTORE-MIB diff --git a/tests/test_translator.py b/tests/test_translator.py index 0af3d3a3..f74470bf 100644 --- a/tests/test_translator.py +++ b/tests/test_translator.py @@ -13,16 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. # ######################################################################## +import json +import logging +import os from unittest import TestCase, mock import mongomock -import yaml - +from pysnmp.hlapi import ( + CommunityData, + ContextData, + SnmpEngine, + UdpTransportTarget, + getCmd, +) +from pysnmp.smi.rfc1902 import ObjectIdentity, ObjectType +from splunk_connect_for_snmp_mib_server.snmp_mib_server import upload_mibs from splunk_connect_for_snmp_mib_server.translator import Translator -import os -import logging -import json -from pysnmp.hlapi import * logger = logging.getLogger(__name__) @@ -81,9 +87,9 @@ def setUp(self): server_config = { "snmp": { "mibs": { - "dir": "mibs/pysnmp", + "dir": "./mibs/pysnmp", "load_list": "lookups/mibs_list.csv", - "mibs_path": "mibs", + "mibs_path": "./mibs", } }, "mongo": { @@ -91,8 +97,8 @@ def setUp(self): "mib": {"database": "files", "collection": "mib_files"}, }, } - #server_config["snmp"]["mibs"]["dir"] = "../mibs/pysnmp" self.my_translator = Translator(server_config) + upload_mibs(server_config) @mongomock.patch() def test_format_trap(self): @@ -145,7 +151,7 @@ def test_format_trap_non_existing_oid(self): value_type = input_var_binds_list[i]["val_type"] oid = input_var_binds_list[i]["oid"] value = input_var_binds_list[i]["val"] - current = f'oid-type{i + 1}="{oid_type}" value{i + 1}-type="{value_type}" {oid}="{value}" value{i + 1}="{value}"' + current = f'oid-type{i+1}="{oid_type}" value{i+1}-type="{value_type}" {oid}="{value}" value{i+1}="{value}"' # these two additional spaces are not an error untranslated += f"{current} " if i < len(input_var_binds_list) - 1: @@ -267,7 +273,7 @@ def test_translate_all_snmp_simulator_data_types(self): "sc4snmp.IF-MIB.ifPhysAddress_2", "sc4snmp.SNMPv2-MIB.sysORID_7", "sc4snmp.TCP-MIB.tcpConnRemAddress_195_218_254_105_51684_194_67_10_226_22", - "sc4snmp.1_3_6_1_2_1_25_3_2_1_6_1025", + "sc4snmp.HOST-RESOURCES-MIB.hrDeviceErrors_1025", "sc4snmp.IF-MIB.ifHighSpeed_2", "sc4snmp.SNMPv2-MIB.sysUpTime_0", "sc4snmp.1_3_6_1_4_1_2021_10_1_6_1", @@ -303,10 +309,16 @@ def test_more_mib_files(self): "val": "sample", "val_type": "DisplayString", }, + { + "oid": "1.3.6.1.2.1.25.1.6.0", + "val": "165", + "val_type": "Gauge32", + }, ] expected_values = [ "sc4snmp.VMSTORE-MIB.mirrorLatency", "sc4snmp.VERITAS-APPLIANCE-MONITORING-MIB.vrtssystemName", + "sc4snmp.HOST-RESOURCES-MIB.hrSystemProcesses_0", ] for i in range(0, len(input_var_binds)):