Add Stats endpoints, settings and management commands

.pre-commit-config.yaml

@@ -107,6 +107,7 @@ repos:
       - id: mypy
         additional_dependencies:
           - types-requests
+          - redis==5.2.0
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.10.1
     hooks:

backend/env.yml

@@ -9,6 +9,7 @@ staging:
   DB_NAME: ${ssm:/crossfeed/staging/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/staging/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/staging/DATABASE_PASSWORD}
+  DJANGO_SECRET: ${ssm:/crossfeed/staging/DJANGO_SECRECT}
   MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
   MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
@@ -44,6 +45,7 @@ staging:
   WORKER_USER_AGENT: ${ssm:/crossfeed/staging/WORKER_USER_AGENT}
   WORKER_SIGNATURE_PUBLIC_KEY: ${ssm:/crossfeed/staging/WORKER_SIGNATURE_PUBLIC_KEY}
   ELASTICSEARCH_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICSEARCH_ENDPOINT}
+  ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICACHE_ENDPOINT}
   REACT_APP_TERMS_VERSION: ${ssm:/crossfeed/staging/REACT_APP_TERMS_VERSION}
   REACT_APP_RANDOM_PASSWORD: ${ssm:/crossfeed/staging/REACT_APP_RANDOM_PASSWORD}
   MATOMO_URL: http://matomo.crossfeed.local
@@ -70,6 +72,7 @@ prod:
   DB_PASSWORD: ${ssm:/crossfeed/prod/DATABASE_PASSWORD}
   MDL_USERNAME: ${ssm:/crossfeed/prod/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/prod/MDL_PASSWORD}
+  DJANGO_SECRET: ${ssm:/crossfeed/prod/DJANGO_SECRECT}
   MDL_NAME: ${ssm:/crossfeed/prod/MDL_NAME}
   MI_ACCOUNT_NAME: ${ssm:/readysetcyber/prod/MI_ACCOUNT_NAME}
   MI_PASSWORD: ${ssm:/readysetcyber/prod/MI_ACCOUNT_PASSWORD}
@@ -95,6 +98,7 @@ prod:
   WORKER_USER_AGENT: ${ssm:/crossfeed/prod/WORKER_USER_AGENT}
   WORKER_SIGNATURE_PUBLIC_KEY: ${ssm:/crossfeed/prod/WORKER_SIGNATURE_PUBLIC_KEY}
   ELASTICSEARCH_ENDPOINT: ${ssm:/crossfeed/prod/ELASTICSEARCH_ENDPOINT}
+  ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/prod/ELASTICACHE_ENDPOINT}
   REACT_APP_TERMS_VERSION: ${ssm:/crossfeed/prod/REACT_APP_TERMS_VERSION}
   REACT_APP_RANDOM_PASSWORD: ${ssm:/crossfeed/prod/REACT_APP_RANDOM_PASSWORD}
   MATOMO_URL: http://matomo.crossfeed.local

backend/requirements.txt

@@ -3,13 +3,16 @@ boto3
 cryptography==38.0.0
 django
 docker
-elasticsearch==7.9.0
+elasticsearch
 fastapi==0.111.0
 mangum==0.17.0
 minio
+mypy
 psycopg2-binary
 PyJWT
 pytest
 pytest-django
+redis==5.2.0
 requests==2.32.3
+types-redis
 uvicorn==0.30.1

backend/src/tasks/functions.yml

@@ -58,3 +58,125 @@ updateScanTaskStatus:
           detail:
             clusterArn:
               - ${file(env.yml):${self:provider.stage}-ecs-cluster, ''}
+
+populateServiceStatsElasticache:
+  handler: src/xfd_django/xfd_api/tasks.populate_ServicesStatscache
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+populatePortStatsElasticache:
+  handler: src/xfd_django/xfd_api/tasks.populate_PortsStatscache
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+populateNumVulnsStatsElasticache:
+  handler: src/xfd_django/xfd_api/tasks.populate_NumVulnerabilitiesStatscache
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+populateLatestVulnsStatsElasticache:
+  handler: src/xfd_django/xfd_api/tasks.populate_LatestVulnerabilitiesCache
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+populateMostCommonVulnerabilityElasticache:
+  handler: src/xfd_django/xfd_api/tasks.populate_MostCommonVulnerabilitiesCache
+
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+populateSeverityCountsCache:
+  handler: src/xfd_django/xfd_api/tasks.populate_SeverityCountsCache
+
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**
+
+populateByOrgCache:
+  handler: src/xfd_django/xfd_api/tasks.populate_VulnerabilitiesByOrgCache
+
+  runtime: python3.11
+  environment:
+    DJANGO_SETTINGS_MODULE: xfd_django.settings
+    REDIS_ENDPOINT: ${env:ELASTICACHE_ENDPOINT}
+  events:
+    - schedule:
+        rate: ron(0 0 * * ? *)  # This triggers the function every day it mightnight
+        enabled: true
+  package:
+    include:
+      - src/**
+      - requirements.txt
+    exclude:
+      - node_modules/**

backend/src/xfd_django/xfd_api/api_methods/organization.py

@@ -19,7 +19,7 @@
 )
 from ..helpers.regionStateMap import REGION_STATE_MAP
 from ..models import Organization, OrganizationTag, Role, Scan, ScanTask, User
-from ..schema_models import organization as organization_schemas
+from ..schema_models import organization_schema
 
 
 def is_valid_uuid(val: str) -> bool:
@@ -330,7 +330,7 @@ def get_all_regions(current_user):
 
 
 def find_or_create_tags(
-    tags: List[organization_schemas.TagSchema],
+    tags: List[organization_schema.TagSchema],
 ) -> List[OrganizationTag]:
     """Find or create organization tags."""
     final_tags = []

backend/src/xfd_django/xfd_api/auth.py

@@ -5,11 +5,12 @@
 import hashlib
 from hashlib import sha256
 import os
-from typing import Optional
+from typing import List, Optional
 from urllib.parse import urlencode
 import uuid
 
 # Third-Party Libraries
+from asgiref.sync import sync_to_async
 from django.conf import settings
 from django.forms.models import model_to_dict
 from fastapi import Depends, HTTPException, Security, status
@@ -19,13 +20,13 @@
 import requests
 
 # from .helpers import user_to_dict
-from .models import ApiKey, Organization, OrganizationTag, Role, User
+from .models import ApiKey, Domain, Organization, OrganizationTag, Role, Service, User
 
 # JWT_ALGORITHM = "RS256"
-JWT_SECRET = os.getenv("JWT_SECRET")
+JWT_SECRET = settings.JWT_SECRET
 SECRET_KEY = settings.SECRET_KEY
-JWT_ALGORITHM = "HS256"
-JWT_TIMEOUT_HOURS = 4
+JWT_ALGORITHM = settings.JWT_ALGORITHM
+JWT_TIMEOUT_HOURS = settings.JWT_TIMEOUT_HOURS
 
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
 api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False)
@@ -87,6 +88,123 @@ def decode_jwt_token(token):
         return None
 
 
+def get_org_memberships(current_user) -> list[str]:
+    """Returns the organization IDs that a user is a member of."""
+    # Check if the user has a 'roles' attribute and it's not None
+
+    roles = Role.objects.filter(userId=current_user)
+    return [role.organizationId.id for role in roles if role.organizationId]
+
+
+async def get_user_domains(user_id: str) -> List[str]:
+    """
+    Retrieves a list of domain names associated with the user's organizations.
+    """
+    try:
+        # Check if the user exists
+        user_exists = await sync_to_async(User.objects.filter(id=user_id).exists)()
+        if not user_exists:
+            return []
+
+        # Fetch organization IDs associated with the user
+        organization_ids_qs = Role.objects.filter(userId__id=user_id).values_list(
+            "organizationId", flat=True
+        )
+        organization_ids = await sync_to_async(lambda qs: list(qs))(organization_ids_qs)
+
+        if not organization_ids:
+            return []
+
+        # Fetch domain names associated with these organizations
+        domain_names_qs = Domain.objects.filter(
+            organizationId__in=organization_ids
+        ).values_list("name", flat=True)
+        domain_list = await sync_to_async(lambda qs: list(qs))(domain_names_qs)
+
+        return domain_list
+    except Exception as e:
+        # Optionally, handle exceptions or return an empty list
+        return []
+
+
+def get_user_service_ids(user_id):
+    """
+    Retrieves service IDs associated with the organizations the user belongs to.
+    """
+    # Get organization IDs the user is a member of
+    organization_ids = Role.objects.filter(userId=user_id).values_list(
+        "organizationId", flat=True
+    )
+
+    # Get domain IDs associated with these organizations
+    domain_ids = Domain.objects.filter(organizationId__in=organization_ids).values_list(
+        "id", flat=True
+    )
+
+    # Get service IDs associated with these domains
+    service_ids = Service.objects.filter(domainId__in=domain_ids).values_list(
+        "id", flat=True
+    )
+
+    return list(map(str, service_ids))  # Convert UUIDs to strings if necessary
+
+
+async def get_user_organization_ids(user_id: str) -> List[str]:
+    try:
+        # Fetch organization IDs associated with the user
+        organization_ids_qs = Role.objects.filter(userId__id=user_id).values_list(
+            "organizationId__id", flat=True
+        )
+        organization_ids = await sync_to_async(list)(organization_ids_qs)
+        return [str(org_id) for org_id in organization_ids]
+    except Exception:
+        return []
+
+
+def get_user_ports(user_id):
+    """
+    Retrieves port numbers associated with the organizations the user belongs to.
+    """
+    # Get organization IDs the user is a member of
+    organization_ids = Role.objects.filter(userId=user_id).values_list(
+        "organizationId", flat=True
+    )
+
+    # Get domain IDs associated with these organizations
+    domain_ids = Domain.objects.filter(organizationId__in=organization_ids).values_list(
+        "id", flat=True
+    )
+
+    # Get ports associated with services of these domains
+    ports = (
+        Service.objects.filter(domainId__in=domain_ids)
+        .values_list("port", flat=True)
+        .distinct()
+    )
+
+    return list(ports)
+
+
+def get_tag_organization_ids(current_user, tag_id: Optional[str] = None) -> list[str]:
+    """Returns the organizations belonging to a tag, if the user can access the tag."""
+    # Check if the user is a global view admin
+    if not is_global_view_admin(current_user):
+        return []
+
+    # Fetch the OrganizationTag and its related organizations
+    tag = (
+        OrganizationTag.objects.prefetch_related("organizations")
+        .filter(id=tag_id)
+        .first()
+    )
+    if tag:
+        # Return a list of organization IDs
+        return [org.id for org in tag.organizations.all()]
+
+    # Return an empty list if tag is not found
+    return []
+
+
 def hash_key(key: str) -> str:
     """
     Helper to hash API key.
@@ -482,14 +600,6 @@ def get_tag_organizations(current_user, tag_id) -> list[str]:
     return []
 
 
-def get_org_memberships(current_user) -> list[str]:
-    """Returns the organization IDs that a user is a member of."""
-    roles = Role.objects.filter(user=current_user)
-    if not roles:
-        return []
-    return [role.organization.id for role in roles if role.organization]
-
-
 def matches_user_region(current_user, user_region_id: str) -> bool:
     """Checks if the current user's region matches the user's region being modified."""