Working on terraform deployment

.github/workflows/build.yml

@@ -10,7 +10,7 @@ on:
     paths-ignore:
       - 'pyproject.toml'
 
-  # Allows you to run this workflow manually from the Actions tab
+  #  Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
 env:
@@ -191,3 +191,31 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
+
+      # Setup Terraform to Deploy
+
+      - name: Configure AWS Credentials as Environment Variables
+        run: echo "AWS_ACCESS_KEY_ID=${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV |
+          echo "AWS_SECRET_ACCESS_KEY=${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}" >> $GITHUB_ENV
+
+      - name: Validate AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-region: us-west-2
+          role-session-name: GitHubActions
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets[format('AWS_ACCESS_KEY_ID_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_SERVICES_{0}', env.TARGET_ENV_UPPERCASE)] }}
+      - run: aws sts get-caller-identity
+
+
+      - name: Validate Terraform
+        run: terraform validate -no-color
+
+      - name: Deploy to venue
+        id: terraform-deploy
+        working-directory: terraform
+        env:
+          AWS_DEFAULT_REGION: us-west-2
+        run: |
+          ./bin/deploy.sh --app-version ${{ env.THE_VERSION }} --tf-venue "sit" --lambda_container_image_uri ${{ env.REGISTRY }}/${{ github.repository }}/${{env.PYPROJECT_NAME}}:sit

terraform/README.md

@@ -0,0 +1,43 @@
+
+# Deploying the HYDROCRON
+
+
+## Dependencies
+There are a handful of dependencies needed to deploy the entire Hydrocron
+
+* Terraform - deployment technology.  >= Terraform v0.12.7
+* AWS CLI - Amazon Web Service command line interface. >= aws-cli/1.11.120
+* python3 environment - tested with python 3.7, needed for packaging the lambda functions.
+
+## Soft dependencies
+These are dependencies for deploying the entire Hydrocron that exist outside of this build.
+
+* access key for NGAP environment
+* Terraform variables defined below
+
+## Terraform Variables
+
+| variable        | Defined In   | Example                                                 | Description |
+| --------------- | ------------ | ------------------------------------------------------- | ----------- |
+| stage           | tf_vars      | sit                                                     | staging environment to which we are deploying |
+| app_name        | tf_vars      | HYDROCRON                                                     | Name of the application being deployed - same for all environments|
+| credentials     | command line | ~/.aws/credentials                                      | AWS credential file to use for authentication |
+| profile         | command line | ngap-services-sit                                       | AWS Profile to use for authentication |
+| docker_tag      | command line | podaac/podaac-cloud/podaac-hydrocron:1.0.0-alpha.3            | Name of docker image and tag as returned from `docker/build-docker.sh`. |
+| vpc_id          | tf_vars      | vpc-04d8fc64e8ce5cca8                                   | VPC Id for use. This is predefined by NGAP. |
+| private_subnets | tf_vars      | ["subnet-0d15606f25bd4047b","subnet-0adee3417fedb7f05"] | private subnets for use within VPC. This is defined by NGAP |
+
+
+## Building the lambda images
+The lambda code needs to be built into a deployable image and uploaded to ECR before running terraform. Normally CI/CD handles this task but if you are trying to run terraform locally it needs to be done manually.
+
+Follow the instructions in the [docker README](../docker/README.md) to build the image
+
+
+## Build and deploy the application
+We use a pre-built docker container to do the deployment (Please do not use local terraform!)
+
+## Destroying the Application
+Similarly, use the pre-built docker container to do the destroy (Please do not use local terraform!)
+
+

terraform/api-specification-templates/hydrocron_aws_api.yml

@@ -0,0 +1,167 @@
+openapi: 3.0.0
+info:
+  title: "Get time series data from SWOT observations for reaches, nodes, and/or lakes"
+  description: "Get time series data from SWOT observations for reaches, nodes, and/or\
+    \ lakes"
+  version: 1.0.0
+servers:
+- url: https://virtserver.swaggerhub.com/hydrocron/HydroAPI/1.0.0
+  description: "Get time series data from SWOT observations for reaches, nodes, and/or\
+    \ lakes"
+paths:
+  /timeseries:
+    get:
+      summary: "Get Timeseries for a particular Reach, Node, or LakeID"
+      description: "Get Timeseries for a particular Reach, Node, or LakeID"
+      operationId: gettimeseries_get
+      parameters:
+      - name: feature
+        in: query
+        description: Data requested for Reach or Node or Lake
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: string
+          enum: [ "Reach", "Lake", "Node"]
+        example: Reach
+      - name: feature_id
+        in: query
+        description: ID of the feature to retrieve in format CBBTTTSNNNNNN (i.e. 74297700000000)
+        required: true
+        style: form
+        explode: true
+        schema:
+          type: string
+        example: 71224100223
+      - name: start_time
+        in: query
+        description: Start time of the timeseries
+        required: true
+        style: form
+        explode: true
+        schema:
+          type: string
+          format: date-time
+        example: 2022-08-04T00:00:00Z
+      - name: end_time
+        in: query
+        description: End time of the timeseries
+        required: true
+        style: form
+        explode: true
+        schema:
+          type: string
+          format: date-time
+        example: 2022-08-23T00:00:00Z
+      - name: output
+        in: query
+        description: Format of the data returned
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: string
+          enum: [ "csv", "geojson"]
+          default: geojson
+        example: geojson
+      - name: fields
+        in: query
+        description: Format of the data returned
+        required: false
+        style: form
+        explode: true
+        schema:
+          type: string
+          default: feature_id, time_str, wse, geometry
+        example: feature_id, time_str, wse, geometry
+      responses:
+        "200":
+          description: OK
+          content:
+            text/csv:
+              schema:
+                type: array
+                items:
+                  type: string
+        "400":
+          description: "400 error. The specified URL is invalid (does not exist)."
+          content:
+            text/csv:
+              schema:
+                type: array
+                items:
+                  type: string
+        "404":
+          description: "404 error. An entry with the specified region was not found."
+          content:
+            text/csv:
+              schema:
+                type: array
+                items:
+                  type: string
+        "413":
+          description: "413 error. Your query has returned is too large."
+          content:
+            text/csv:
+              schema:
+                type: array
+                items:
+                  type: string
+      x-openapi-router-controller: hydrocron_api.controllers.timeseries
+
+      x-amazon-apigateway-integration:
+        uri: ${hydrocron_api_lambda_arn_timeseries}
+        responses:
+          default:
+            statusCode: "200"
+            responseTemplates:
+              application/json: |
+                #set($inputRoot = $input.path('$'))
+                #if($inputRoot.toString().contains('206 PARTIAL CONTENT'))
+                  #set($context.responseOverride.status = 206)
+                #end
+                $input.json('$')
+          ^400.*:
+            statusCode: "400"
+            responseTemplates:
+              application/json: |-
+                {
+                  "error" : "$input.path('$.errorMessage')"
+                }
+          ^404.*:
+            statusCode: "404"
+            responseTemplates:
+              application/json: |-
+                {
+                  "error" : "$input.path('$.errorMessage')"
+                }
+          ^413.*:
+            statusCode: "413"
+            responseTemplates:
+              application/json: |-
+                {
+                  "error" : "$input.path('$.errorMessage')"
+                }
+          ^[^1-5].*:
+            statusCode: "500"
+            responseTemplates:
+              application/json: |-
+                {
+                  "error" : "$input.path('$.errorMessage')"
+                }
+        requestTemplates:
+          application/json: |-
+            {
+              "body": {
+                "exact":"$input.params('exact')",
+                "region": "$input.params('region')",
+                "page_number": "$input.params('page_number')" ,
+                "page_size": "$input.params('page_size')" ,
+                "polygon_format": "$input.params('polygon_format')"
+              }
+            }
+        passthroughBehavior: when_no_templates
+        httpMethod: POST
+        contentHandling: CONVERT_TO_TEXT
+        type: aws

terraform/bin/deploy.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+set -Eexo pipefail
+
+# Read in args from command line
+
+POSITIONAL=()
+while [[ $# -gt 0 ]]
+do
+key="$1"
+
+case $key in
+    --ticket)
+    ticket="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    --app-version)
+    app_version="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    --lambda_container_image_uri)
+    lambda_container_image_uri="$2"
+    shift # past argument
+    shift # past value
+    ;;
+    -v|--tf-venue)
+    tf_venue="$2"
+    case $tf_venue in
+     sit|uat|ops) ;;
+     *)
+        echo "tf_venue must be sit, uat, or ops"
+        exit 1;;
+    esac
+    shift # past argument
+    shift # past value
+    ;;
+    *)    # unknown option
+    POSITIONAL+=("$1") # save it in an array for later
+    shift # past argument
+    ;;
+esac
+done
+set -- "${POSITIONAL[@]}" # restore positional parameters
+
+# https://www.terraform.io/docs/commands/environment-variables.html#tf_in_automation
+TF_IN_AUTOMATION=true
+
+# Terraform initialization
+terraform init -reconfigure -input=false -backend-config="bucket=podaac-services-${tf_venue}-terraform" -backend-config="profile=ngap-service-${tf_venue}"
+
+if [[ "${ticket}" ]]; then
+  set +e
+  terraform workspace new "${ticket}"
+  set -e
+  terraform workspace select "${ticket}"
+else
+  terraform workspace select default
+fi
+
+terraform plan -input=false -var-file=tfvars/"${tf_venue}".tfvars -var="credentials=~/.aws/credentials" -var="profile=ngap-service-${tf_venue}" -var="app_version=${app_version}" -var="lambda_container_image_uri"=${lambda_container_image_uri} -out="tfplan"
+
+# Apply the plan that was created
+terraform apply -input=false -auto-approve tfplan