Merge branch 'develop' into rapidray12-utilities-1

cisagov · Feb 12, 2025 · fee6e3f · fee6e3f
2 parents bf7040f + 9834bb3
commit fee6e3f
Show file tree

Hide file tree

Showing 359 changed files with 15,899 additions and 94,375 deletions.
diff --git a/backend/Dockerfile.pe b/backend/Dockerfile.pe
@@ -7,7 +7,7 @@ COPY ./package* ./
 
 COPY src ./src
 
-RUN apt update && apt install git zlib1g-dev
+RUN apt update && apt install -y git zlib1g-dev
 
 RUN apt-get update && apt-get install -y jq
 

diff --git a/backend/Makefile b/backend/Makefile
@@ -9,6 +9,18 @@ syncdb:
 syncdb-populate:
 	docker compose exec backend python manage.py syncdb --populate
 
+syncdb-dangerously-force:
+	docker compose exec backend python manage.py syncdb --dangerouslyforce
+
+
+# Synchronize and populate the database
+syncmdl:
+	docker compose exec backend python manage.py syncmdl
+
+syncmdl-dangerously-force:
+	docker compose exec backend python manage.py syncmdl --dangerouslyforce
+
+
 # Pytest
 # i.e. make pytest FILE=xfd_api/tests/test_domain.py
 pytest:

diff --git a/backend/env.yml b/backend/env.yml
@@ -6,6 +6,7 @@ dev:
   REGION: us-east-1
 
 staging-cd:
+  IS_DMZ: 1
   REGION: us-east-1
   ENDPOINT_TYPE: REGIONAL
   COGNITO_URL: https://cognito-idp.us-east-1.amazonaws.com
@@ -24,6 +25,8 @@ staging-cd:
   MDL_USERNAME: ${ssm:/crossfeed/staging/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/staging/MDL_PASSWORD}
   MDL_NAME: ${ssm:/crossfeed/staging/MDL_NAME}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/staging/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/staging/POSTGRES_SCAN_PASSWORD}
   MI_ACCOUNT_NAME: ${ssm:/readysetcyber/staging/MI_ACCOUNT_NAME}
   MI_PASSWORD: ${ssm:/readysetcyber/staging/MI_ACCOUNT_PASSWORD}
   PE_DB_NAME: ${ssm:/crossfeed/staging/PE_DB_NAME}
@@ -40,7 +43,7 @@ staging-cd:
   LOGIN_GOV_BASE_URL: ${ssm:/crossfeed/staging/LOGIN_GOV_BASE_URL}
   LOGIN_GOV_JWT_KEY: ${ssm:/crossfeed/staging/LOGIN_GOV_JWT_KEY}
   LOGIN_GOV_ISSUER: ${ssm:/crossfeed/staging/LOGIN_GOV_ISSUER}
-  DOMAIN: ${ssm:/crossfeed/staging/DOMAIN}
+  DOMAIN: ${ssm:/crossfeed/staging-cd/DOMAIN}
   CERT_DOMAIN: api.staging-cd.crossfeed.cyber.dhs.gov
   FARGATE_SG_ID: ${ssm:/crossfeed/staging/WORKER_SG_ID}
   FARGATE_SUBNET_ID: ${ssm:/crossfeed/staging/WORKER_SUBNET_ID}
@@ -78,6 +81,7 @@ staging-cd:
   REACT_APP_COGNITO_CALLBACK_URL: https://staging-cd.crossfeed.cyber.dhs.gov/okta-callback
 
 integration:
+  IS_DMZ: 1
   REGION: us-east-1
   ENDPOINT_TYPE: REGIONAL
   COGNITO_URL: https://cognito-idp.us-east-1.amazonaws.com
@@ -95,6 +99,8 @@ integration:
   MDL_USERNAME: ${ssm:/crossfeed/integration/MDL_USERNAME}
   MDL_PASSWORD: ${ssm:/crossfeed/integration/MDL_PASSWORD}
   MDL_NAME: ${ssm:/crossfeed/integration/MDL_NAME}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/integration/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/integration/POSTGRES_SCAN_PASSWORD}
   MI_ACCOUNT_NAME: ${ssm:/readysetcyber/integration/MI_ACCOUNT_NAME}
   MI_PASSWORD: ${ssm:/readysetcyber/integration/MI_ACCOUNT_PASSWORD}
   JWT_SECRET: ${ssm:/crossfeed/integration/APP_JWT_SECRET}
@@ -142,6 +148,7 @@ integration:
   ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/integration/ELASTICACHE_ENDPOINT}
 
 staging:
+  IS_DMZ: 0
   REGION: us-gov-east-1
   ENDPOINT_TYPE: PRIVATE
   COGNITO_URL: https://cognito-idp.us-gov-west-1.amazonaws.com
@@ -157,6 +164,8 @@ staging:
   DB_NAME: ${ssm:/crossfeed/staging/DATABASE_NAME}
   DB_USERNAME: ${ssm:/crossfeed/staging/DATABASE_USER}
   DB_PASSWORD: ${ssm:/crossfeed/staging/DATABASE_PASSWORD}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/staging/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/staging/POSTGRES_SCAN_PASSWORD}
   JWT_SECRET: ${ssm:/crossfeed/staging/APP_JWT_SECRET}
   JWT_ALGORITHM: ${ssm:/crossfeed/staging/JWT_ALGORITHM}
   JWT_TIMEOUT_HOURS: ${ssm:/crossfeed/staging/JWT_TIMEOUT_HOURS}
@@ -185,6 +194,8 @@ staging:
   ELASTICSEARCH_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICSEARCH_ENDPOINT}
   REACT_APP_TERMS_VERSION: ${ssm:/crossfeed/staging/REACT_APP_TERMS_VERSION}
   MATOMO_URL: http://matomo.cfs.lz.us-cert.gov
+  MATOMO_DB_HOST: ${ssm:/crossfeed/staging/MATOMO_DATABASE_HOST}
+  MATOMO_DB_PASSWORD: ${ssm:/crossfeed/staging/MATOMO_DATABASE_PASSWORD}
   EXPORT_BUCKET_NAME: cisa-crossfeed-staging-exports
   PE_API_URL: ${ssm:/crossfeed/staging/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-staging-reports
@@ -204,6 +215,7 @@ staging:
   ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/staging/ELASTICACHE_ENDPOINT}
 
 prod:
+  IS_DMZ: 0
   REGION: us-gov-east-1
   ENDPOINT_TYPE: PRIVATE
   COGNITO_URL: https://cognito-idp.us-gov-west-1.amazonaws.com
@@ -223,6 +235,8 @@ prod:
   MDL_PASSWORD: ${ssm:/crossfeed/prod/MDL_PASSWORD}
   DJANGO_SECRET: ${ssm:/crossfeed/prod/DJANGO_SECRECT}
   MDL_NAME: ${ssm:/crossfeed/prod/MDL_NAME}
+  POSTGRES_SCAN_USER: ${ssm:/crossfeed/prod/POSTGRES_SCAN_USER}
+  POSTGRES_SCAN_PASSWORD: ${ssm:/crossfeed/prod/POSTGRES_SCAN_PASSWORD}
   JWT_SECRET: ${ssm:/crossfeed/prod/APP_JWT_SECRET}
   JWT_ALGORITHM: ${ssm:/crossfeed/prod/JWT_ALGORITHM}
   JWT_TIMEOUT_HOURS: ${ssm:/crossfeed/prod/JWT_TIMEOUT_HOURS}
@@ -231,7 +245,7 @@ prod:
   LOGIN_GOV_JWT_KEY: ${ssm:/crossfeed/prod/LOGIN_GOV_JWT_KEY}
   LOGIN_GOV_ISSUER: ${ssm:/crossfeed/prod/LOGIN_GOV_ISSUER}
   DOMAIN: ${ssm:/crossfeed/prod/DOMAIN}
-  CERT_DOMAIN: staging.crossfeed.cyber.dhs.gov
+  CERT_DOMAIN: crossfeed.cyber.dhs.gov
   FARGATE_SG_ID: ${ssm:/crossfeed/prod/WORKER_SG_ID}
   FARGATE_SUBNET_ID: ${ssm:/crossfeed/prod/WORKER_SUBNET_ID}
   FARGATE_MAX_CONCURRENCY: 300
@@ -252,6 +266,8 @@ prod:
   ELASTICACHE_ENDPOINT: ${ssm:/crossfeed/prod/ELASTICACHE_ENDPOINT}
   REACT_APP_TERMS_VERSION: ${ssm:/crossfeed/prod/REACT_APP_TERMS_VERSION}
   MATOMO_URL: http://matomo.cfs.lz.us-cert.gov
+  MATOMO_DB_HOST: ${ssm:/crossfeed/prod/MATOMO_DATABASE_HOST}
+  MATOMO_DB_PASSWORD: ${ssm:/crossfeed/prod/MATOMO_DATABASE_PASSWORD}
   EXPORT_BUCKET_NAME: cisa-crossfeed-prod-exports
   PE_API_URL: ${ssm:/crossfeed/prod/PE_API_URL}
   REPORTS_BUCKET_NAME: cisa-crossfeed-prod-reports

diff --git a/backend/jest.config.js b/backend/jest.config.js
diff --git a/backend/mock.js b/backend/mock.js
diff --git a/backend/requirements.txt b/backend/requirements.txt
@@ -17,6 +17,7 @@ click==8.1.7
 cryptography==38.0.0
 dill==0.3.9
 Django==5.1.4
+django-netfields==1.3.2
 dnspython==2.7.0
 docker==7.1.0
 elasticsearch==7.9.0
@@ -42,6 +43,7 @@ minio==7.2.12
 multidict==6.1.0
 mypy==1.13.0
 mypy-extensions==1.0.0
+netaddr==1.3.0
 orjson==3.10.12
 packaging==24.2
 pika==1.3.2
@@ -57,6 +59,7 @@ pydantic_core==2.27.1
 Pygments==2.18.0
 PyJWT==2.10.1
 pylint==3.3.2
+PyMySQL==1.1.1
 pytest==8.3.4
 pytest-django==4.9.0
 pytest-env==1.1.5
@@ -78,6 +81,7 @@ starlette==0.37.2
 tomlkit==0.13.2
 typer==0.15.1
 types-cffi==1.16.0.20240331
+types-PyMySQL==1.1.0.20241103
 types-pyOpenSSL==24.1.0.20240722
 types-redis==4.6.0.20241004
 types-setuptools==75.6.0.20241126