Merge pull request #256 from aws-solutions/morjoan-dev

Morjoan dev

.github/workflows/notify-workflow.yml

@@ -0,0 +1,61 @@
+name: Send Artifacts Workflow
+env:
+  REGION: us-east-1
+on: 
+  workflow_run:
+    workflows: ["Push Workflow"]
+    types: [completed]
+jobs:
+  notify-job:
+    runs-on: ubuntu-latest
+    name: Send workflow information to endpoint
+    environment: notify_env
+    permissions:
+      id-token: write
+      contents: read
+      actions: read
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}    
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Install required system packages
+        run: |
+          pip install --upgrade --force-reinstall -r deployment/requirements.txt 2> error.txt
+          if [ -s error.txt ]; then
+            echo "ERROR: System package installation failed."
+            cat error.txt
+            exit 1
+          fi
+      - name: Set up environment variables
+        run: |
+          export WORKFLOW_RUN_ID=${{ github.event.workflow_run.id }} 
+          export COMMIT_ID=${{ github.event.workflow_run.head_sha }}
+          export BRANCH=${{ github.event.workflow_run.head_branch }}
+          export WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
+          export NOTIFICATION_ENDPOINT=${{ secrets.ENDPOINT }}
+          export VERSION=${{ secrets.VERSION }}
+
+          echo WORKFLOW_RUN_ID=$WORKFLOW_RUN_ID >> $GITHUB_ENV
+          echo COMMIT_ID=$COMMIT_ID >> $GITHUB_ENV
+          echo NOTIFICATION_ENDPOINT=$NOTIFICATION_ENDPOINT >> $GITHUB_ENV
+          echo BRANCH=$BRANCH >> $GITHUB_ENV
+          echo WORKFLOW_NAME=$WORKFLOW_NAME >> $GITHUB_ENV
+          echo VERSION=$VERSION >> $GITHUB_ENV
+      - name: Determine pipeline type
+        run: |
+          if [ $BRANCH == "main" ]; then
+            export PIPELINE_TYPE="release"
+          else
+            export PIPELINE_TYPE="feature"
+          fi
+          echo PIPELINE_TYPE=$PIPELINE_TYPE >> $GITHUB_ENV
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.NOTIFY_ROLE }}
+          aws-region: ${{ env.REGION }}
+          role-duration-seconds: 900
+      - name: Invoke endpoint
+        run: |
+          cd deployment
+          python end-workflow-notification.py

.github/workflows/pull-request-workflow.yml

@@ -3,7 +3,6 @@ name: Pull Request Workflow
 # global environment variables to all jobs
 env:
   REGION: us-east-1
-  SOLUTION_NAME: aws-media-services-application-mapper
   DIST_OUTPUT_BUCKET: msam-pr-build
 
 on: 
@@ -123,7 +122,7 @@ jobs:
       - name: Run build script
         run: |
           cd deployment
-          ./build-s3-dist.sh $DIST_OUTPUT_BUCKET $SOLUTION_NAME $VERSION
+          ./build-s3-dist.sh $DIST_OUTPUT_BUCKET ${{ github.event.repository.name }} $VERSION
       # - name: Run unit tests
       #   run: |
       #     cd deployment

.github/workflows/push-workflow.yml

@@ -3,10 +3,8 @@ name: Push Workflow
 # global environment variables to all jobs
 env:
   REGION: us-east-1
-  SOLUTION_NAME: aws-media-services-application-mapper
   DIST_OUTPUT_BUCKET: msam-dev
   REGIONS_TO_DEPLOY: "us-west-2 us-east-1"
-  RELEASE_BRANCH: refs/heads/main
 
 on:
   push:
@@ -145,7 +143,7 @@ jobs:
       - name: Run build script
         run: |
           cd deployment
-          ./build-s3-dist.sh $DIST_OUTPUT_BUCKET $SOLUTION_NAME $VERSION
+          ./build-s3-dist.sh $DIST_OUTPUT_BUCKET ${{ github.event.repository.name }} $VERSION
       # - name: Run unit tests
       #   run: |
       #     cd deployment
@@ -232,7 +230,7 @@ jobs:
       - name: Upload to S3
         run: |
           cd deployment
-          ./deploy.sh -b $DIST_OUTPUT_BUCKET -s $SOLUTION_NAME -v $VERSION -r "$REGIONS_TO_DEPLOY" -a none -t dev
+          ./deploy.sh -b $DIST_OUTPUT_BUCKET -s ${{ github.event.repository.name }} -v $VERSION -r "$REGIONS_TO_DEPLOY" -a none -t dev
       - name: Zip up regional and global assets
         run: |
           cd deployment
@@ -243,34 +241,3 @@ jobs:
         with:
           name: zipped-regional-global-assets
           path: deployment/*.zip  
-
-  # install msam for test if pushing changes to main branch
-  main-branch-deploy-cfn:
-    name: Install MSAM for test
-    runs-on: ubuntu-latest
-    environment: push_env
-    permissions:
-      actions: read
-      id-token: write
-      contents: read
-    needs:
-      [
-        cfn-lint-job,
-        cfn-nag-job        
-      ]
-    if: github.ref == 'refs/heads/main'
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@master
-        with:
-          role-to-assume: ${{ secrets.MSAM_ROLE_ARN }}
-          aws-region: ${{ env.REGION }}
-      - name: Generate version out of commit ID
-        run: |
-          echo "VERSION=${GITHUB_SHA::7}" >> $GITHUB_ENV
-      - name: Create CloudFormation stack based on last job's build
-        run: |
-          TEMPLATE="https://$DIST_OUTPUT_BUCKET-$REGION.s3.amazonaws.com/$SOLUTION_NAME/latest/$SOLUTION_NAME-release.template"
-          echo $TEMPLATE
-          response="$( aws cloudformation create-stack --stack-name MSAM-test-main-$VERSION --template-url $TEMPLATE --region $REGION --capabilities CAPABILITY_IAM CAPABILITY_NAMED_IAM CAPABILITY_AUTO_EXPAND --disable-rollback )"
-          echo $response

deployment/build-s3-dist.sh

@@ -157,7 +157,7 @@ mv dynamodb_resource.zip $build_dist_dir/dynamodb_resource_$STAMP.zip
 cd $source_dir/html
 echo "updating browser app build stamp"
 cp -f js/app/build-tmp.js js/app/build.js
-sed -i -e "s/DEV_0_0_0/$STAMP/g" js/app/build.js
+sed -i -e "s/VERSION/$VERSION/g" js/app/build.js
 zip -q -r $build_dist_dir/msam-web-$STAMP.zip *
 rm -f js/app/build.js-e
 

deployment/end-workflow-notification.py

@@ -0,0 +1,39 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+This module sends notification about the end of a workflow run
+and its associated artifacts.
+"""
+
+import os
+import requests
+from aws_requests_auth.boto_utils import BotoAWSRequestsAuth
+from urllib.parse import urlparse
+
+API_REGION = os.environ.get('AWS_DEFAULT_REGION')
+NOTIFICATION_ENDPOINT = os.environ.get('NOTIFICATION_ENDPOINT')
+
+PAYLOAD = {}
+PAYLOAD['solution_name'] = os.environ.get('GITHUB_REPOSITORY').split('/')[1]
+PAYLOAD['branch'] = os.environ.get('BRANCH')
+PAYLOAD['workflow_name'] = os.environ.get('WORKFLOW_NAME')
+PAYLOAD['commit_id'] = os.environ.get('COMMIT_ID')
+PAYLOAD['workflow_run_id'] = os.environ.get('WORKFLOW_RUN_ID')
+PAYLOAD['version'] = os.environ.get('VERSION')
+PAYLOAD['pipeline_type'] = os.environ.get('PIPELINE_TYPE')
+def main():
+    parsed = urlparse(NOTIFICATION_ENDPOINT)
+    auth = BotoAWSRequestsAuth(aws_host=parsed.netloc,
+                               aws_region=API_REGION,
+                               aws_service='execute-api')
+    print(PAYLOAD)
+    response = requests.post(NOTIFICATION_ENDPOINT, json=PAYLOAD, auth=auth, timeout=25)
+    print(response.text)
+    if response.status_code != 200:
+        return 1
+    else:
+        return 0
+
+if __name__ == "__main__":
+    main()

deployment/requirements.txt

@@ -11,3 +11,4 @@ pylint
 requests
 stringcase
 testresources
+aws-requests-auth

source/events/cloudwatch_alarm.py

@@ -14,8 +14,9 @@
 from botocore.config import Config
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(user_agent=f"aws-media-services-applications-mapper/{STAMP}/cloudwatch_alarm.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 ALARMS_TABLE_NAME = os.environ["ALARMS_TABLE_NAME"]
 TABLE_REGION = os.environ["EVENTS_TABLE_REGION"]

source/events/media_events.py

@@ -18,8 +18,9 @@
 from jsonpath_ng import parse
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(user_agent=f"aws-media-services-applications-mapper/{STAMP}/media_events.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 DYNAMO_REGION_NAME=os.environ["EVENTS_TABLE_REGION"]
 DYNAMO_RESOURCE = boto3.resource('dynamodb', region_name=DYNAMO_REGION_NAME, config=MSAM_BOTO3_CONFIG)

source/events/msam-events-release.template

@@ -2,6 +2,13 @@
     "AWSTemplateFormatVersion": "2010-09-09",
     "Transform": "AWS::Serverless-2016-10-31",
     "Description": "Media Services Application Mapper Event Capture %%VERSION%%",
+    "Mappings": {
+        "SolutionId": {
+            "UserAgent": {
+                "Extra": "AwsSolution/SO0048/%%VERSION%%"
+            }
+        }
+    },
     "Resources": {
         "Collector": {
             "Type": "AWS::Serverless::Function",
@@ -44,7 +51,8 @@
                         },
                         "ITEM_TTL": {
                             "Ref": "ItemTTL"
-                        }
+                        },
+                        "SOLUTION_ID": { "Fn::FindInMap": ["SolutionId", "UserAgent", "Extra"] }
                     }
                 },
                 "Events": {
@@ -105,7 +113,8 @@
                         },
                         "ALARMS_TABLE_NAME": {
                             "Ref": "AlarmsTableName"
-                        }
+                        },
+                        "SOLUTION_ID": { "Fn::FindInMap": ["SolutionId", "UserAgent", "Extra"] }
                     }
                 },
                 "Events": {

source/html/js/app/build-tmp.js

@@ -3,11 +3,11 @@
 
 define(["app/server"], function(server) {
 
-    var stamp = "DEV_0_0_0";
+    var version = "VERSION";
 
     return {
-        "get_timestamp": function() {
-            return stamp;
+        "get_version": function() {
+            return version;
         }
     };
 

source/html/js/app/tools/build_numbers.js

@@ -14,7 +14,7 @@ define(["jquery", "app/api_check", "app/build", "app/connections"],
                 var current_connection = connections.get_current();
                 var endpoint = current_connection[0];
                 var api_key = current_connection[1];
-                var app_stamp = build.get_timestamp();
+                var app_stamp = build.get_version();
                 check.ping(endpoint, api_key).then(function(response) {
                     var api_stamp = Number.parseInt(response.buildstamp);
                     var browser_stamp = Number.parseInt(app_stamp);

source/msam/.chalice/config.json

@@ -16,6 +16,8 @@
         "EVENTS_TABLE_NAME": "media-services-application-mapper-events",
         "LAYOUT_TABLE_NAME": "media-services-application-mapper-layout",
         "SETTINGS_TABLE_NAME": "media-services-application-mapper-settings",
-        "CLOUDWATCH_EVENTS_TABLE_NAME": "media-services-application-mapper-cloudwatchevents"
+        "CLOUDWATCH_EVENTS_TABLE_NAME": "media-services-application-mapper-cloudwatchevents",
+        "SOLUTION_ID": "AwsSolution/SO0048/%%VERSION%%",
+        "VERSION": "%%VERSION%%"
     }
 }

source/msam/app.py

@@ -404,7 +404,7 @@ def ping():
     """
     API entry point to test the API key authentication and retrieve the build timestamp.
     """
-    return {"message": "pong", "buildstamp": os.environ["BUILD_STAMP"]}
+    return {"message": "pong", "buildstamp": os.environ["BUILD_STAMP"], "version": os.environ["VERSION"]}
 
 
 @app.schedule(Rate(NODE_UPDATE_RATE_MINUTES, unit=Rate.MINUTES))

source/msam/chalicelib/cache.py

@@ -16,8 +16,9 @@
 CONTENT_TABLE_NAME = os.environ["CONTENT_TABLE_NAME"]
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(user_agent=f"aws-media-services-applications-mapper/{STAMP}/cache.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 
 def cached_by_service(service):

source/msam/chalicelib/channels.py

@@ -18,9 +18,9 @@
 CHANNELS_TABLE_NAME = os.environ["CHANNELS_TABLE_NAME"]
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(
-    user_agent=f"aws-media-services-applications-mapper/{STAMP}/channels.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 # DynamoDB
 DYNAMO_RESOURCE = boto3.resource("dynamodb", config=MSAM_BOTO3_CONFIG)

source/msam/chalicelib/cloudwatch.py

@@ -22,9 +22,9 @@
 CLOUDWATCH_EVENTS_TABLE_NAME = os.environ["CLOUDWATCH_EVENTS_TABLE_NAME"]
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(
-    user_agent=f"aws-media-services-applications-mapper/{STAMP}/cloudwatch.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 
 def update_alarm_records(region_name, alarm, subscriber_arns):

source/msam/chalicelib/content.py

@@ -16,8 +16,9 @@
 CONTENT_TABLE_NAME = os.environ["CONTENT_TABLE_NAME"]
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(user_agent=f"aws-media-services-applications-mapper/{STAMP}/content.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 def put_ddb_items(items):
     """

source/msam/chalicelib/layout.py

@@ -18,9 +18,9 @@
 LAYOUT_TABLE_NAME = os.environ["LAYOUT_TABLE_NAME"]
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(
-    user_agent=f"aws-media-services-applications-mapper/{STAMP}/layout.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 # DynamoDB
 DYNAMO_RESOURCE = boto3.resource("dynamodb", config=MSAM_BOTO3_CONFIG)

source/msam/chalicelib/nodes.py

@@ -21,9 +21,10 @@
 # TTL provided via CloudFormation
 CACHE_ITEM_TTL = int(os.environ["CACHE_ITEM_TTL"])
 
-STAMP = os.environ["BUILD_STAMP"]
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
 # used to handle throttling, be very patient and back off a lot if needed
-MSAM_BOTO3_CONFIG = Config(retries={'max_attempts': 15}, user_agent=f"aws-media-services-applications-mapper/{STAMP}/nodes.py")
+MSAM_BOTO3_CONFIG = Config(retries={'max_attempts': 15}, **USER_AGENT_EXTRA)
 
 def update_regional_ddb_items(region_name):
     """

source/msam/chalicelib/periodic.py

@@ -26,8 +26,9 @@
 CONTENT_TABLE_NAME = os.environ["CONTENT_TABLE_NAME"]
 
 # user-agent config
-STAMP = os.environ["BUILD_STAMP"]
-MSAM_BOTO3_CONFIG = Config(user_agent=f"aws-media-services-applications-mapper/{STAMP}/periodic.py")
+SOLUTION_ID = os.environ['SOLUTION_ID']
+USER_AGENT_EXTRA = {"user_agent_extra": SOLUTION_ID}
+MSAM_BOTO3_CONFIG = Config(**USER_AGENT_EXTRA)
 
 SSM_LOG_GROUP_NAME = "MSAM/SSMRunCommand"