Adds validation checksum management and model

backend/src/xfd_django/xfd_api/api_methods/sync.py

@@ -13,6 +13,7 @@
 
 from ..helpers.s3_client import S3Client
 from ..utils.csv_utils import convert_csv_to_json, create_checksum
+from ..utils.validation import save_validation_checksum
 
 
 # Helper function
@@ -32,6 +33,7 @@ async def sync_post(sync_body, request: Request):
             return {"status": 500}
 
         # Use MinIO client to save CSV data to S3
+        save_validation_checksum(generated_checksum, "DMZ INGEST")
         s3_client = S3Client()
         cursor_header = headers.get("x-cursor")
         start_bound, end_bound = -1, -2

backend/src/xfd_django/xfd_api/tasks/vulnScanningSync.py

@@ -28,6 +28,7 @@
     save_ticket_to_datalake,
     save_vuln_scan,
 )
+from xfd_api.utils.validation import save_validation_checksum
 from xfd_mini_dl.models import Cidr, Organization, Sector
 
 
@@ -306,8 +307,13 @@ def main():
                 "Content-Type": "application/json",
                 "Authorization": os.environ.get("DMZ_API_KEY"),
             }
+            print("Sending chunk to /sync")
+            save_validation_checksum(checksum, "LZ PUSH TO DMZ")
             response = requests.post(
-                os.environ.get("DMZ_SYNC_ENDPOINT"), json=body, headers=headers
+                os.environ.get("DMZ_SYNC_ENDPOINT"),
+                json=body,
+                headers=headers,
+                timeout=60,
             )
             if response.status_code == 200:
                 print("CSV Succesfully sent to /sync")

backend/src/xfd_django/xfd_api/utils/csv_utils.py

@@ -6,23 +6,23 @@
 
 # Standard Python Libraries
 import csv
-from hashlib import sha256
 from io import StringIO
 import json
 from typing import Any, Dict, List
+import zlib
 
 
 def create_checksum(data: str) -> str:
-    """Generate a SHA-256 checksum for a given string.
+    """Generate a CRC32 checksum for a given string.
 
     Args:
         data (str): The input string.
 
     Returns:
-        str: The SHA-256 hash of the input string.
+        str: The CRC32 hash of the input string as a hex string.
     """
-    hash_object = sha256(data.encode("utf-8"))
-    return hash_object.hexdigest()
+    crc = zlib.crc32(data.encode("utf-8"))
+    return format(crc & 0xFFFFFFFF, "08x")
 
 
 def json_to_csv(json_array: List[Dict[str, Any]]) -> str:

backend/src/xfd_django/xfd_api/utils/validation.py

@@ -0,0 +1,33 @@
+"""Validation utilities for syncing operations."""
+
+# Standard Python Libraries
+from datetime import datetime
+
+# Third-Party Libraries
+from xfd_mini_dl.models import SyncChecksum
+
+
+def save_validation_checksum(checksum: str, type: str) -> bool:
+    """
+    Save a validation checksum to the data lake.
+
+    This function attempts to create a new SyncChecksum record in the database
+    with the provided checksum and sync type. If the operation is successful,
+    it returns True. If an exception occurs, it logs the error and returns False.
+
+    Args:
+        checksum (str): The checksum value to store.
+        type (str): The type of sync operation associated with the checksum.
+
+    Returns:
+        bool: True if the checksum was successfully saved, False otherwise.
+    """
+    try:
+        SyncChecksum.objects.create(
+            checksum=checksum, sync_type=type, sync_date=datetime.now()
+        )
+        return True
+    except Exception as e:
+        # Optionally, log the exception if needed
+        print(f"Error saving checksum: {e}")
+        return False

backend/src/xfd_django/xfd_mini_dl/models.py

@@ -5640,6 +5640,32 @@ class Meta:
         unique_together = (("cpe_product_name", "version_number"),)
 
 
+class SyncChecksum(models.Model):
+    """Define SyncChecksums model."""
+
+    sync_checksums_uid = models.UUIDField(
+        primary_key=True,
+        default=uuid.uuid1,
+        help_text="PK: Unique identifier for a sync checksum object.",
+    )
+    sync_type = models.TextField(
+        blank=True, null=True, help_text="Type of sync that was performed."
+    )
+    checksum = models.TextField(
+        blank=True, null=True, help_text="Checksum of the data that was synced."
+    )
+    sync_date = models.DateTimeField(
+        blank=True, null=True, help_text="Datetime the sync was performed."
+    )
+
+    class Meta:
+        """Set SyncChecksums model metadata."""
+
+        app_label = app_label_name
+        managed = manage_db
+        db_table = "sync_checksums"
+
+
 # # THese are all views, so they shouldn't be generated via the ORM
 
 # # This should be a view not a table