Element84 STAC Query Support

rdwatch/core/tasks/__init__.py

@@ -88,30 +88,37 @@ def get_siteobservation_images_task(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
     worldview_source: Literal['cog', 'nitf'] | None = 'cog',
 ) -> None:
-    capture_count = 0
-    for constellation in baseConstellations:
-        capture_count += get_siteobservations_images(
-            self,
-            site_eval_id=site_eval_id,
-            baseConstellation=constellation,
-            force=force,
-            dayRange=dayRange,
-            no_data_limit=no_data_limit,
-            overrideDates=overrideDates,
-            scale=scale,
-            bboxScale=bboxScale,
-            worldview_source=worldview_source,
-        )
-    fetching_task = SatelliteFetching.objects.get(site_id=site_eval_id)
-    fetching_task.status = SatelliteFetching.Status.COMPLETE
-    if capture_count == 0:
-        fetching_task.error = 'No Captures found'
-    fetching_task.celery_id = ''
-    fetching_task.save()
+    try:
+        capture_count = 0
+        for constellation in baseConstellations:
+            capture_count += get_siteobservations_images(
+                self,
+                site_eval_id=site_eval_id,
+                baseConstellation=constellation,
+                force=force,
+                dayRange=dayRange,
+                no_data_limit=no_data_limit,
+                overrideDates=overrideDates,
+                scale=scale,
+                bboxScale=bboxScale,
+                worldview_source=worldview_source,
+            )
+        fetching_task = SatelliteFetching.objects.get(site_id=site_eval_id)
+        fetching_task.status = SatelliteFetching.Status.COMPLETE
+        if capture_count == 0:
+            fetching_task.error = 'No Captures found'
+        fetching_task.celery_id = ''
+        fetching_task.save()
+    except Exception as e:
+        logger.warning(f'EXCEPTION: {e}')
+        fetching_task = SatelliteFetching.objects.get(site_id=site_eval_id)
+        fetching_task.error = f'Error: {e}'
+        fetching_task.status = SatelliteFetching.Status.ERROR
+        fetching_task.save()
 
 
 def get_siteobservations_images(
@@ -122,7 +129,7 @@ def get_siteobservations_images(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
     worldview_source: Literal['cog', 'nitf'] | None = 'cog',
 ) -> None:
@@ -352,11 +359,11 @@ def get_siteobservations_images(
         if capture_timestamp not in found_timestamps.keys():
             # we need to add a new image into the structure
             bytes = None
-            if worldview_source == 'cog':
+            if baseConstellation == 'WV' and worldview_source == 'cog':
                 bytes = get_worldview_processed_visual_bbox(
                     capture, max_bbox, 'PNG', scale
                 )
-            elif worldview_source == 'nitf':
+            elif baseConstellation == 'WV' and worldview_source == 'nitf':
                 bytes = get_worldview_nitf_bbox(capture, max_bbox, 'PNG', scale)
             else:
                 bytes = get_raster_bbox(capture.uri, max_bbox, 'PNG', scale)
@@ -559,7 +566,7 @@ def generate_site_images(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
     worldview_source: Literal['cog', 'nitf'] | None = 'cog',
 ):
@@ -579,6 +586,7 @@ def generate_site_images(
             # Otherwise, if the task exists but is *not* running, set the status
             # to running and kick off the task
             fetching_task.status = SatelliteFetching.Status.RUNNING
+            fetching_task.timestamp = datetime.now()
             fetching_task.save()
         else:
             fetching_task = SatelliteFetching.objects.create(
@@ -609,7 +617,7 @@ def generate_site_images_for_evaluation_run(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ):
     sites = SiteEvaluation.objects.filter(configuration=model_run_id)

rdwatch/core/utils/images.py

@@ -92,9 +92,9 @@ def get_range_captures(
     timebuffer: timedelta,
     worldView: Literal['cog', 'nitf'] | None,
 ):
-    if worldView == 'cog':
+    if constellation == 'WV' and worldView == 'cog':
         captures = get_worldview_captures(timestamp, bbox, timebuffer)
-    elif worldView == 'nitf':
+    elif constellation == 'WV' and worldView == 'nitf':
         captures = get_worldview_nitf_captures(timestamp, bbox, timebuffer)
     else:
         captures = list(get_bands(constellation, timestamp, bbox, timebuffer))
@@ -114,7 +114,7 @@ def fetch_boundbox_image(
     timestamp: datetime,
     constellation: str,
     worldView: Literal['cog', 'nitf'] | None = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
 ):
     timebuffer = timedelta(days=1)
     try:
@@ -133,9 +133,9 @@ def fetch_boundbox_image(
     if len(captures) == 0:
         return None
     closest_capture = min(captures, key=lambda band: abs(band.timestamp - timestamp))
-    if worldView == 'cog':
+    if worldView == 'cog' and constellation == 'wv':
         bytes = get_worldview_processed_visual_bbox(closest_capture, bbox, 'PNG', scale)
-    elif worldView == 'nitf':
+    elif worldView == 'nitf' and constellation == 'wv':
         bytes = get_worldview_nitf_bbox(closest_capture, bbox, 'PNG', scale)
     else:
         bytes = get_raster_bbox(closest_capture.uri, bbox, 'PNG', scale)

rdwatch/core/utils/raster_tile.py

@@ -27,7 +27,7 @@ def get_raster_bbox(
     uri: str,
     bbox: tuple[float, float, float, float],
     format='PNG',
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
 ) -> bytes:
     with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR'):
         if uri.startswith('https://sentinel-cogs.s3.us-west-2.amazonaws.com'):

rdwatch/core/utils/satellite_bands.py

@@ -4,6 +4,8 @@
 from datetime import datetime, timedelta
 from typing import cast
 
+from django.conf import settings
+
 from rdwatch.core.models.lookups import CommonBand, ProcessingLevel
 from rdwatch.core.utils.stac_search import stac_search
 
@@ -22,6 +24,18 @@ class Band:
     collection: str
 
 
+COLLECTIONS: list[str] = []
+
+if settings.STAC_URL:
+    COLLECTIONS = ['sentinel-2-c1-l2a', 'sentinel-2-l2a', 'landsat-c2-l2']
+elif settings.settings.ACCENTURE_VERSION is not None:
+    COLLECTIONS = [
+        f'ta1-s2-acc-{settings.ACCENTURE_VERSION}',
+        f'ta1-ls-acc-{settings.ACCENTURE_VERSION}',
+        f'ta1-pd-acc-{settings.ACCENTURE_VERSION}',
+    ]
+
+
 def get_bands(
     constellation: str,
     timestamp: datetime,
@@ -62,23 +76,8 @@ def get_bands(
 
         cloudcover = 0
         match feature:
-            case {'collection': 'landsat-c2l1' | 'sentinel-s2-l1c'}:
-                level, _ = ProcessingLevel.objects.get_or_create(
-                    slug='1C',
-                    defaults={'description': 'top of atmosphere radiance'},
-                )
             case {'collection': collection}:
-                if (
-                    collection
-                    in (
-                        'landsat-c2l2-sr',
-                        'sentinel-s2-l2a',
-                        'sentinel-s2-l2a-cogs',
-                    )
-                    or collection.startswith('ta1-s2-acc-')
-                    or collection.startswith('ta1-ls-acc-')
-                    or collection.startswith('ta1-pd-acc-')
-                ):
+                if collection in COLLECTIONS:
                     level, _ = ProcessingLevel.objects.get_or_create(
                         slug='2A',
                         defaults={'description': 'surface reflectance'},

rdwatch/core/utils/stac_search.py

@@ -56,10 +56,18 @@ def _fmt_time(time: datetime):
     'PL': [],
 }
 
-if settings.ACCENTURE_VERSION is not None:
+STAC_URL = ''
+STAC_HEADERS = {}
+if settings.STAC_URL:
+    COLLECTIONS['S2'].append(['sentinel-2-c1-l2a', 'sentinel-2-l2a'])
+    COLLECTIONS['L8'].append('landsat-c2-l2')
+    STAC_URL = settings.STAC_URL
+elif settings.settings.ACCENTURE_VERSION is not None:
     COLLECTIONS['S2'].append(f'ta1-s2-acc-{settings.ACCENTURE_VERSION}')
     COLLECTIONS['L8'].append(f'ta1-ls-acc-{settings.ACCENTURE_VERSION}')
     COLLECTIONS['PL'].append(f'ta1-pd-acc-{settings.ACCENTURE_VERSION}')
+    STAC_URL = settings.SMART_STAC_URL
+    STAC_HEADERS = {'x-api-key': settings.SMART_STAC_KEY}
 
 
 def stac_search(
@@ -68,28 +76,26 @@ def stac_search(
     bbox: tuple[float, float, float, float],
     timebuffer: timedelta | None = None,
 ) -> Results:
-    stac_catalog = Client.open(
-        # Use SMART program server instead of public server
-        # (https://earth-search.aws.element84.com/v0/search)
-        settings.SMART_STAC_URL,
-        headers={'x-api-key': settings.SMART_STAC_KEY},
-    )
+    stac_catalog = Client.open(STAC_URL, headers=STAC_HEADERS)
 
     if timebuffer is not None:
         min_time = timestamp - timebuffer
         max_time = timestamp + timebuffer
         time_str = f'{_fmt_time(min_time)}/{_fmt_time(max_time)}'
     else:
         time_str = f'{_fmt_time(timestamp)}Z'
-
+    if source not in COLLECTIONS:
+        logger.warning(
+            'Source {source} not in Collections: {COLLECTIONS} returning empty list'
+        )
+        return {'features': [], 'links': []}
     results = stac_catalog.search(
         method='GET',
         bbox=bbox,
         datetime=time_str,
         collections=COLLECTIONS[source],
         limit=100,
     )
-
     # TODO: return `results` directly instead of converting to a dict.
     # Before that can happen, the callers need to be updated to handle
     # an `ItemSearch` object.

rdwatch/core/utils/worldview_nitf/raster_tile.py

@@ -56,7 +56,7 @@ def get_worldview_nitf_bbox(
     capture: WorldViewNITFCapture,
     bbox: tuple[float, float, float, float],
     format='PNG',
-    scale: Literal['default', 'bits'] = 'default',
+    scale: Literal['default', 'bits'] = 'bits',
 ) -> bytes:
     with rasterio.Env(
         GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',

rdwatch/core/utils/worldview_processed/raster_tile.py

@@ -57,7 +57,7 @@ def get_worldview_processed_visual_bbox(
     capture: WorldViewProcessedCapture,
     bbox: tuple[float, float, float, float],
     format='PNG',
-    scale: Literal['default', 'bits'] = 'default',
+    scale: Literal['default', 'bits'] = 'bits',
 ) -> bytes:
     with rasterio.Env(
         GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',
@@ -72,8 +72,8 @@ def get_worldview_processed_visual_bbox(
     ):
         startTime = time.time()
         if not capture.panuri:
+            logger.warning(f'Image URI: {capture.uri}')
             with Reader(input=capture.uri) as img:
-                logger.warning(f'Image URI: {capture.uri}')
                 logger.warning(f'Base Info Time: {time.time() - startTime}')
                 rgb = img.part(bbox)
                 logger.warning(f'RGB Download Time: {time.time() - startTime}')

rdwatch/core/utils/worldview_processed/stac_search.py

@@ -71,7 +71,6 @@ def worldview_search(
         time_str = f'{_fmt_time(min_time)}/{_fmt_time(max_time)}'
     else:
         time_str = f'{_fmt_time(timestamp)}Z'
-
     results = stac_catalog.search(
         method='GET',
         bbox=bbox,

rdwatch/core/views/site_observation.py

@@ -172,7 +172,6 @@ def site_observations(request: HttpRequest, evaluation_id: UUID4):
             ),
         )
     )
-    print(image_queryset)
 
     for image in image_queryset['results']:
         image['image'] = default_storage.url(image['image'])
@@ -212,20 +211,14 @@ class GenerateImagesSchema(Schema):
     noData: int = 50
     overrideDates: None | list[str] = None
     force: bool = False
-    scale: Literal['default', 'bits', 'custom'] = 'default'
+    scale: Literal['default', 'bits', 'custom'] = 'bits'
     scaleNum: None | list[int] = None
     bboxScale: None | float = 1.2
 
     @root_validator
     def validate_worldview_source(cls, values: dict[str, Any]):
-        print(values)
         if 'WV' in values['constellation'] and values['worldviewSource'] is None:
             raise ValueError('worldviewSource is required for WV constellation')
-        elif (
-            'WV' not in values['constellation']
-            and values['worldviewSource'] is not None
-        ):
-            raise ValueError('worldviewSource is only for WV constellation')
         return values
 
 

rdwatch/scoring/tasks/__init__.py

@@ -53,7 +53,7 @@ def get_siteobservation_images_task(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ) -> None:
     capture_count = 0
@@ -85,7 +85,7 @@ def get_siteobservations_images(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ) -> None:
     # Ensure we are using ints for the DayRange and no_data_limit
@@ -428,7 +428,7 @@ def generate_site_images(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ):
     with transaction.atomic():
@@ -478,7 +478,7 @@ def generate_site_images_for_evaluation_run(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ):
     try:

rdwatch/scoring/views/site_observation.py

@@ -39,7 +39,7 @@ class GenerateImagesSchema(Schema):
     noData: int = 50
     overrideDates: None | list[str] = None
     force: bool = False
-    scale: Literal['default', 'bits', 'custom'] = 'default'
+    scale: Literal['default', 'bits', 'custom'] = 'bits'
     scaleNum: None | list[int] = None
     bboxScale: None | float = 1.2
 

rdwatch/settings.py

@@ -158,6 +158,7 @@ def DATABASES(self):
         environ_required=True, environ_prefix=_ENVIRON_PREFIX
     )
 
+    STAC_URL = values.URLValue(environ_required=False, environ_prefix=_ENVIRON_PREFIX)
     SMART_STAC_URL = values.URLValue(
         environ_required=True, environ_prefix=_ENVIRON_PREFIX
     )

template.env

@@ -1,6 +1,8 @@
 # URL pointing to the SMART STAC server
 RDWATCH_SMART_STAC_URL=
 
+RDWATCH_STAC_URL="https://earth-search.aws.element84.com/v1/"
+
 # API key for the SMART STAC server
 RDWATCH_SMART_STAC_KEY=
 

vue/src/components/ImagesDownloadDialog.vue

@@ -32,7 +32,7 @@ const showAdvanced = ref(false);
 const force =ref(false);
 const customDateRange = ref(false);
 const scaleOptions = ref(['default', 'bits', 'custom'])
-const scale: Ref<'default' | 'bits' | 'custom'> = ref('default')
+const scale: Ref<'default' | 'bits' | 'custom'> = ref('bits')
 const scaleNums: Ref<[number, number]> = ref([0, 10000])
 const bboxScale: Ref<number> = ref(1.2);
 const validForm = ref(true);