Element84 STAC Query Support

rdwatch/core/tasks/__init__.py

@@ -88,30 +88,37 @@ def get_siteobservation_images_task(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
     worldview_source: Literal['cog', 'nitf'] | None = 'cog',
 ) -> None:
-    capture_count = 0
-    for constellation in baseConstellations:
-        capture_count += get_siteobservations_images(
-            self,
-            site_eval_id=site_eval_id,
-            baseConstellation=constellation,
-            force=force,
-            dayRange=dayRange,
-            no_data_limit=no_data_limit,
-            overrideDates=overrideDates,
-            scale=scale,
-            bboxScale=bboxScale,
-            worldview_source=worldview_source,
-        )
-    fetching_task = SatelliteFetching.objects.get(site_id=site_eval_id)
-    fetching_task.status = SatelliteFetching.Status.COMPLETE
-    if capture_count == 0:
-        fetching_task.error = 'No Captures found'
-    fetching_task.celery_id = ''
-    fetching_task.save()
+    try:
+        capture_count = 0
+        for constellation in baseConstellations:
+            capture_count += get_siteobservations_images(
+                self,
+                site_eval_id=site_eval_id,
+                baseConstellation=constellation,
+                force=force,
+                dayRange=dayRange,
+                no_data_limit=no_data_limit,
+                overrideDates=overrideDates,
+                scale=scale,
+                bboxScale=bboxScale,
+                worldview_source=worldview_source,
+            )
+        fetching_task = SatelliteFetching.objects.get(site_id=site_eval_id)
+        fetching_task.status = SatelliteFetching.Status.COMPLETE
+        if capture_count == 0:
+            fetching_task.error = 'No Captures found'
+        fetching_task.celery_id = ''
+        fetching_task.save()
+    except Exception as e:
+        logger.warning(f'EXCEPTION: {e}')
+        fetching_task = SatelliteFetching.objects.get(site_id=site_eval_id)
+        fetching_task.error = f'Error: {e}'
+        fetching_task.status = SatelliteFetching.Status.ERROR
+        fetching_task.save()
 
 
 def get_siteobservations_images(
@@ -122,7 +129,7 @@ def get_siteobservations_images(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
     worldview_source: Literal['cog', 'nitf'] | None = 'cog',
 ) -> None:
@@ -352,11 +359,11 @@ def get_siteobservations_images(
         if capture_timestamp not in found_timestamps.keys():
             # we need to add a new image into the structure
             bytes = None
-            if worldview_source == 'cog':
+            if baseConstellation == 'WV' and worldview_source == 'cog':
                 bytes = get_worldview_processed_visual_bbox(
                     capture, max_bbox, 'PNG', scale
                 )
-            elif worldview_source == 'nitf':
+            elif baseConstellation == 'WV' and worldview_source == 'nitf':
                 bytes = get_worldview_nitf_bbox(capture, max_bbox, 'PNG', scale)
             else:
                 bytes = get_raster_bbox(capture.uri, max_bbox, 'PNG', scale)
@@ -559,7 +566,7 @@ def generate_site_images(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
     worldview_source: Literal['cog', 'nitf'] | None = 'cog',
 ):
@@ -579,6 +586,7 @@ def generate_site_images(
             # Otherwise, if the task exists but is *not* running, set the status
             # to running and kick off the task
             fetching_task.status = SatelliteFetching.Status.RUNNING
+            fetching_task.timestamp = datetime.now()
             fetching_task.save()
         else:
             fetching_task = SatelliteFetching.objects.create(
@@ -609,7 +617,7 @@ def generate_site_images_for_evaluation_run(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ):
     sites = SiteEvaluation.objects.filter(configuration=model_run_id)

rdwatch/core/utils/images.py

@@ -92,9 +92,9 @@ def get_range_captures(
     timebuffer: timedelta,
     worldView: Literal['cog', 'nitf'] | None,
 ):
-    if worldView == 'cog':
+    if constellation == 'WV' and worldView == 'cog':
         captures = get_worldview_captures(timestamp, bbox, timebuffer)
-    elif worldView == 'nitf':
+    elif constellation == 'WV' and worldView == 'nitf':
         captures = get_worldview_nitf_captures(timestamp, bbox, timebuffer)
     else:
         captures = list(get_bands(constellation, timestamp, bbox, timebuffer))
@@ -114,7 +114,7 @@ def fetch_boundbox_image(
     timestamp: datetime,
     constellation: str,
     worldView: Literal['cog', 'nitf'] | None = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
 ):
     timebuffer = timedelta(days=1)
     try:
@@ -133,9 +133,9 @@ def fetch_boundbox_image(
     if len(captures) == 0:
         return None
     closest_capture = min(captures, key=lambda band: abs(band.timestamp - timestamp))
-    if worldView == 'cog':
+    if worldView == 'cog' and constellation == 'wv':
         bytes = get_worldview_processed_visual_bbox(closest_capture, bbox, 'PNG', scale)
-    elif worldView == 'nitf':
+    elif worldView == 'nitf' and constellation == 'wv':
         bytes = get_worldview_nitf_bbox(closest_capture, bbox, 'PNG', scale)
     else:
         bytes = get_raster_bbox(closest_capture.uri, bbox, 'PNG', scale)

rdwatch/core/utils/raster_tile.py

@@ -15,7 +15,17 @@ def get_raster_tile(uri: str, z: int, x: int, y: int) -> bytes:
                 s3_uri = 's3://sentinel-cogs/' + uri[49:]
                 with Reader(input=s3_uri) as cog:
                     img = cog.tile(x, y, z, tilesize=512)
-                    img.rescale(in_range=((0, 10000),))
+                    # TODO Rescaling is off for element84 img.rescale(in_range=((0, 10000),))
+                    # Using bit scaling instead
+                    stats = cog.statistics()
+                    low = 0
+                    high = 10000
+                    if 'b1' in stats.keys():
+                        stats_json = stats['b1']
+                        low = stats_json['percentile_2']
+                        high = stats_json['percentile_98']
+                    img.rescale(in_range=((low, high),))
+
                     return img.render(img_format='WEBP')
         with Reader(input=uri) as cog:
             img = cog.tile(x, y, z, tilesize=512)
@@ -27,7 +37,7 @@ def get_raster_bbox(
     uri: str,
     bbox: tuple[float, float, float, float],
     format='PNG',
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
 ) -> bytes:
     with rasterio.Env(GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR'):
         if uri.startswith('https://sentinel-cogs.s3.us-west-2.amazonaws.com'):

rdwatch/core/utils/satellite_bands.py

@@ -22,6 +22,9 @@ class Band:
     collection: str
 
 
+COLLECTIONS: list[str] = ['sentinel-2-c1-l2a', 'sentinel-2-l2a', 'landsat-c2-l2']
+
+
 def get_bands(
     constellation: str,
     timestamp: datetime,
@@ -62,23 +65,8 @@ def get_bands(
 
         cloudcover = 0
         match feature:
-            case {'collection': 'landsat-c2l1' | 'sentinel-s2-l1c'}:
-                level, _ = ProcessingLevel.objects.get_or_create(
-                    slug='1C',
-                    defaults={'description': 'top of atmosphere radiance'},
-                )
             case {'collection': collection}:
-                if (
-                    collection
-                    in (
-                        'landsat-c2l2-sr',
-                        'sentinel-s2-l2a',
-                        'sentinel-s2-l2a-cogs',
-                    )
-                    or collection.startswith('ta1-s2-acc-')
-                    or collection.startswith('ta1-ls-acc-')
-                    or collection.startswith('ta1-pd-acc-')
-                ):
+                if collection in COLLECTIONS:
                     level, _ = ProcessingLevel.objects.get_or_create(
                         slug='2A',
                         defaults={'description': 'surface reflectance'},

rdwatch/core/utils/stac_search.py

@@ -4,8 +4,6 @@
 
 from pystac_client import Client
 
-from django.conf import settings
-
 logger = logging.getLogger(__name__)
 
 
@@ -52,14 +50,11 @@ def _fmt_time(time: datetime):
 
 COLLECTIONS: dict[str, list[str]] = {
     'L8': [],
-    'S2': [],
+    'S2': ['sentinel-2-c1-l2a', 'sentinel-2-l2a'],
     'PL': [],
 }
 
-if settings.ACCENTURE_VERSION is not None:
-    COLLECTIONS['S2'].append(f'ta1-s2-acc-{settings.ACCENTURE_VERSION}')
-    COLLECTIONS['L8'].append(f'ta1-ls-acc-{settings.ACCENTURE_VERSION}')
-    COLLECTIONS['PL'].append(f'ta1-pd-acc-{settings.ACCENTURE_VERSION}')
+STAC_URL = 'https://earth-search.aws.element84.com/v1/'
 
 
 def stac_search(
@@ -68,12 +63,7 @@ def stac_search(
     bbox: tuple[float, float, float, float],
     timebuffer: timedelta | None = None,
 ) -> Results:
-    stac_catalog = Client.open(
-        # Use SMART program server instead of public server
-        # (https://earth-search.aws.element84.com/v0/search)
-        settings.SMART_STAC_URL,
-        headers={'x-api-key': settings.SMART_STAC_KEY},
-    )
+    stac_catalog = Client.open(STAC_URL)
 
     if timebuffer is not None:
         min_time = timestamp - timebuffer

rdwatch/core/utils/worldview_nitf/raster_tile.py

@@ -56,7 +56,7 @@ def get_worldview_nitf_bbox(
     capture: WorldViewNITFCapture,
     bbox: tuple[float, float, float, float],
     format='PNG',
-    scale: Literal['default', 'bits'] = 'default',
+    scale: Literal['default', 'bits'] = 'bits',
 ) -> bytes:
     with rasterio.Env(
         GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',

rdwatch/core/utils/worldview_processed/raster_tile.py

@@ -57,7 +57,7 @@ def get_worldview_processed_visual_bbox(
     capture: WorldViewProcessedCapture,
     bbox: tuple[float, float, float, float],
     format='PNG',
-    scale: Literal['default', 'bits'] = 'default',
+    scale: Literal['default', 'bits'] = 'bits',
 ) -> bytes:
     with rasterio.Env(
         GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR',
@@ -72,8 +72,8 @@ def get_worldview_processed_visual_bbox(
     ):
         startTime = time.time()
         if not capture.panuri:
+            logger.warning(f'Image URI: {capture.uri}')
             with Reader(input=capture.uri) as img:
-                logger.warning(f'Image URI: {capture.uri}')
                 logger.warning(f'Base Info Time: {time.time() - startTime}')
                 rgb = img.part(bbox)
                 logger.warning(f'RGB Download Time: {time.time() - startTime}')

rdwatch/core/utils/worldview_processed/stac_search.py

@@ -71,7 +71,6 @@ def worldview_search(
         time_str = f'{_fmt_time(min_time)}/{_fmt_time(max_time)}'
     else:
         time_str = f'{_fmt_time(timestamp)}Z'
-
     results = stac_catalog.search(
         method='GET',
         bbox=bbox,

rdwatch/core/views/site_observation.py

@@ -172,7 +172,6 @@ def site_observations(request: HttpRequest, evaluation_id: UUID4):
             ),
         )
     )
-    print(image_queryset)
 
     for image in image_queryset['results']:
         image['image'] = default_storage.url(image['image'])
@@ -212,20 +211,14 @@ class GenerateImagesSchema(Schema):
     noData: int = 50
     overrideDates: None | list[str] = None
     force: bool = False
-    scale: Literal['default', 'bits', 'custom'] = 'default'
+    scale: Literal['default', 'bits', 'custom'] = 'bits'
     scaleNum: None | list[int] = None
     bboxScale: None | float = 1.2
 
     @root_validator
     def validate_worldview_source(cls, values: dict[str, Any]):
-        print(values)
         if 'WV' in values['constellation'] and values['worldviewSource'] is None:
             raise ValueError('worldviewSource is required for WV constellation')
-        elif (
-            'WV' not in values['constellation']
-            and values['worldviewSource'] is not None
-        ):
-            raise ValueError('worldviewSource is only for WV constellation')
         return values
 
 

rdwatch/scoring/tasks/__init__.py

@@ -53,7 +53,7 @@ def get_siteobservation_images_task(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ) -> None:
     capture_count = 0
@@ -85,7 +85,7 @@ def get_siteobservations_images(
     dayRange=14,
     no_data_limit=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ) -> None:
     # Ensure we are using ints for the DayRange and no_data_limit
@@ -428,7 +428,7 @@ def generate_site_images(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ):
     with transaction.atomic():
@@ -478,7 +478,7 @@ def generate_site_images_for_evaluation_run(
     dayRange=14,
     noData=50,
     overrideDates: None | list[datetime, datetime] = None,
-    scale: Literal['default', 'bits'] | list[int] = 'default',
+    scale: Literal['default', 'bits'] | list[int] = 'bits',
     bboxScale: float = BboxScaleDefault,
 ):
     try:

rdwatch/scoring/views/site_observation.py

@@ -39,7 +39,7 @@ class GenerateImagesSchema(Schema):
     noData: int = 50
     overrideDates: None | list[str] = None
     force: bool = False
-    scale: Literal['default', 'bits', 'custom'] = 'default'
+    scale: Literal['default', 'bits', 'custom'] = 'bits'
     scaleNum: None | list[int] = None
     bboxScale: None | float = 1.2
 

vue/src/components/ImagesDownloadDialog.vue

@@ -19,10 +19,10 @@ const emit = defineEmits<{
     (e: "cancel"): void;
 }>();
 
-const constellationChoices = ref(['S2', 'WV', 'L8', 'PL'])
+const constellationChoices = ref(['S2', 'WV',]) // TODO: Image Download HotFix ref(['S2', 'WV', 'L8', 'PL'])
 const selectedConstellation: Ref<Constellation[]> = ref(['WV']);
-const worldviewSourceChoices = computed<string[] | null>(() => selectedConstellation.value.includes('WV') ? ['cog', 'nitf'] : null);
-const selectedWorldviewSource = ref<'cog' | 'nitf'>('cog');
+const worldviewSourceChoices = computed<string[] | null>(() => selectedConstellation.value.includes('WV') ? ['nitf'] : null); // TODO Image Download Hotfix: add back in cog
+const selectedWorldviewSource = ref<'nitf'>('nitf'); // // TODO: Image Download HotFix ref<'cog' | 'nitf'>('cog');
 const dayRange = ref(14);
 const noData = ref(50)
 const overrideDates: Ref<[string, string]> = ref([
@@ -31,8 +31,8 @@ const overrideDates: Ref<[string, string]> = ref([
 const showAdvanced = ref(false);
 const force =ref(false);
 const customDateRange = ref(false);
-const scaleOptions = ref(['default', 'bits', 'custom'])
-const scale: Ref<'default' | 'bits' | 'custom'> = ref('default')
+const scaleOptions = ref(['default', 'bits', 'custom']);
+const scale: Ref<'default' | 'bits' | 'custom'> = ref('bits');
 const scaleNums: Ref<[number, number]> = ref([0, 10000])
 const bboxScale: Ref<number> = ref(1.2);
 const validForm = ref(true);

vue/src/components/SettingsPanel.vue

@@ -318,6 +318,7 @@ watch(databaseSource, () => {
         <v-col cols="8">
           <v-checkbox
             v-model="worldViewImagery"
+            :disabled="true"
             label="WorldView Imagery"
             density="compact"
           />

vue/src/mapstyle/satellite-image.ts

@@ -90,8 +90,8 @@ export const setSatelliteTimeStamp = (state: { filters: MapFilters, satellite: S
     // Lets try to get the closes timestamp that is less than the current time.
     // Try each date in the list until one works.
     for (const date of baseList) {
-      const timeStamp = date.toISOString().substring(0,19);
-      const currentIndex = state.satellite.satelliteTimeList.findIndex((item) => item.timestamp === timeStamp);
+      const timeStamp = date.toISOString().substring(0, 19);
+      const currentIndex = state.satellite.satelliteTimeList.findIndex((item) => item.timestamp.substring(0, 19) === timeStamp);
       if (currentIndex === -1) {
         continue;
       }