From 2970d5bc524adb1e8036ed58ba57d1fd57813e1f Mon Sep 17 00:00:00 2001 From: Janson Bunce Date: Thu, 20 Feb 2025 10:57:23 -0800 Subject: [PATCH] Adds validation checksum management and model --- .../xfd_django/xfd_api/api_methods/sync.py | 2 ++ .../xfd_api/tasks/vulnScanningSync.py | 8 ++++- .../src/xfd_django/xfd_api/utils/csv_utils.py | 10 +++--- .../xfd_django/xfd_api/utils/validation.py | 33 +++++++++++++++++++ backend/src/xfd_django/xfd_mini_dl/models.py | 26 +++++++++++++++ 5 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 backend/src/xfd_django/xfd_api/utils/validation.py diff --git a/backend/src/xfd_django/xfd_api/api_methods/sync.py b/backend/src/xfd_django/xfd_api/api_methods/sync.py index e7c305ea4..a8c8267c0 100644 --- a/backend/src/xfd_django/xfd_api/api_methods/sync.py +++ b/backend/src/xfd_django/xfd_api/api_methods/sync.py @@ -13,6 +13,7 @@ from ..helpers.s3_client import S3Client from ..utils.csv_utils import convert_csv_to_json, create_checksum +from ..utils.validation import save_validation_checksum # Helper function @@ -32,6 +33,7 @@ async def sync_post(sync_body, request: Request): return {"status": 500} # Use MinIO client to save CSV data to S3 + save_validation_checksum(generated_checksum, "DMZ INGEST") s3_client = S3Client() cursor_header = headers.get("x-cursor") start_bound, end_bound = -1, -2 diff --git a/backend/src/xfd_django/xfd_api/tasks/vulnScanningSync.py b/backend/src/xfd_django/xfd_api/tasks/vulnScanningSync.py index 50547b1ec..a2bba7aa1 100644 --- a/backend/src/xfd_django/xfd_api/tasks/vulnScanningSync.py +++ b/backend/src/xfd_django/xfd_api/tasks/vulnScanningSync.py @@ -28,6 +28,7 @@ save_ticket_to_datalake, save_vuln_scan, ) +from xfd_api.utils.validation import save_validation_checksum from xfd_mini_dl.models import Cidr, Organization, Sector @@ -306,8 +307,13 @@ def main(): "Content-Type": "application/json", "Authorization": os.environ.get("DMZ_API_KEY"), } + print("Sending chunk to /sync") + save_validation_checksum(checksum, "LZ PUSH TO DMZ") response = requests.post( - os.environ.get("DMZ_SYNC_ENDPOINT"), json=body, headers=headers + os.environ.get("DMZ_SYNC_ENDPOINT"), + json=body, + headers=headers, + timeout=60, ) if response.status_code == 200: print("CSV Succesfully sent to /sync") diff --git a/backend/src/xfd_django/xfd_api/utils/csv_utils.py b/backend/src/xfd_django/xfd_api/utils/csv_utils.py index 0cd45c391..05f56cc93 100644 --- a/backend/src/xfd_django/xfd_api/utils/csv_utils.py +++ b/backend/src/xfd_django/xfd_api/utils/csv_utils.py @@ -6,23 +6,23 @@ # Standard Python Libraries import csv -from hashlib import sha256 from io import StringIO import json from typing import Any, Dict, List +import zlib def create_checksum(data: str) -> str: - """Generate a SHA-256 checksum for a given string. + """Generate a CRC32 checksum for a given string. Args: data (str): The input string. Returns: - str: The SHA-256 hash of the input string. + str: The CRC32 hash of the input string as a hex string. """ - hash_object = sha256(data.encode("utf-8")) - return hash_object.hexdigest() + crc = zlib.crc32(data.encode("utf-8")) + return format(crc & 0xFFFFFFFF, "08x") def json_to_csv(json_array: List[Dict[str, Any]]) -> str: diff --git a/backend/src/xfd_django/xfd_api/utils/validation.py b/backend/src/xfd_django/xfd_api/utils/validation.py new file mode 100644 index 000000000..aac7c31a6 --- /dev/null +++ b/backend/src/xfd_django/xfd_api/utils/validation.py @@ -0,0 +1,33 @@ +"""Validation utilities for syncing operations.""" + +# Standard Python Libraries +from datetime import datetime + +# Third-Party Libraries +from xfd_mini_dl.models import SyncChecksum + + +def save_validation_checksum(checksum: str, type: str) -> bool: + """ + Save a validation checksum to the data lake. + + This function attempts to create a new SyncChecksum record in the database + with the provided checksum and sync type. If the operation is successful, + it returns True. If an exception occurs, it logs the error and returns False. + + Args: + checksum (str): The checksum value to store. + type (str): The type of sync operation associated with the checksum. + + Returns: + bool: True if the checksum was successfully saved, False otherwise. + """ + try: + SyncChecksum.objects.create( + checksum=checksum, sync_type=type, sync_date=datetime.now() + ) + return True + except Exception as e: + # Optionally, log the exception if needed + print(f"Error saving checksum: {e}") + return False diff --git a/backend/src/xfd_django/xfd_mini_dl/models.py b/backend/src/xfd_django/xfd_mini_dl/models.py index 0a18b249c..243b0f548 100644 --- a/backend/src/xfd_django/xfd_mini_dl/models.py +++ b/backend/src/xfd_django/xfd_mini_dl/models.py @@ -5640,6 +5640,32 @@ class Meta: unique_together = (("cpe_product_name", "version_number"),) +class SyncChecksum(models.Model): + """Define SyncChecksums model.""" + + sync_checksums_uid = models.UUIDField( + primary_key=True, + default=uuid.uuid1, + help_text="PK: Unique identifier for a sync checksum object.", + ) + sync_type = models.TextField( + blank=True, null=True, help_text="Type of sync that was performed." + ) + checksum = models.TextField( + blank=True, null=True, help_text="Checksum of the data that was synced." + ) + sync_date = models.DateTimeField( + blank=True, null=True, help_text="Datetime the sync was performed." + ) + + class Meta: + """Set SyncChecksums model metadata.""" + + app_label = app_label_name + managed = manage_db + db_table = "sync_checksums" + + # # THese are all views, so they shouldn't be generated via the ORM # # This should be a view not a table