Feature/fabiano commits

Makefile

@@ -22,15 +22,15 @@ build: build-common ## - build a debug binary to the current platform (windows,
 
 .PHONY: format-check
 format-check: ## - check files format using gofmt
-	@ ./_support/scripts/ci.sh fmtcheck
+	@ ./_support/scripts/ci.sh fmtCheck
 
 .PHONY: format-check
 format: ## - apply golang file format using gofmt
 	@ ./_support/scripts/ci.sh format
 
 .PHONY: test
 test: build-common ## - execute go test command for unit and mocked tests
-	@ ./_support/scripts/ci.sh unittest
+	@ ./_support/scripts/ci.sh unitTest
 
 .PHONY: integration-test
 integration-test: ## - execute go test command for integration tests (aws credentials needed)

README.md

@@ -1,22 +1,22 @@
 # VMWare Go KCL v2
 
 ![technology Go](https://img.shields.io/badge/technology-go-blue.svg)
-[![Go Report Card](https://goreportcard.com/badge/github.com/vmware/vmware-go-kcl)](https://goreportcard.com/report/github.com/vmware/vmware-go-kcl)
+[![Go Report Card](https://goreportcard.com/badge/github.com/vmware/vmware-go-kcl-v2)](https://goreportcard.com/report/github.com/vmware/vmware-go-kcl-v2)
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
-[![vmware-go-kcl-v2](https://github.com/fafg/vmware-go-kcl/actions/workflows/vmware-go-kcl-v2-ci.yml/badge.svg)](https://github.com/fafg/vmware-go-kcl/actions/workflows/vmware-go-kcl-v2-ci.yml)
+[![vmware-go-kcl-v2](https://github.com/fafg/vmware-go-kcl-v2/actions/workflows/vmware-go-kcl-v2-ci.yml/badge.svg)](https://github.com/fafg/vmware-go-kcl-v2/actions/workflows/vmware-go-kcl-v2-ci.yml)
 
 ## Overview
 
-VMware-Go-KCL is a native open-source Go library for Amazon Kinesis Data Stream (KDS) consumption. It allows developers 
-to program KDS consumers in lightweight Go language and still take advantage of the features presented by the native 
+VMware-Go-KCL-V2 is a native open-source Go library for Amazon Kinesis Data Stream (KDS) consumption. It allows developers
+to program KDS consumers in lightweight Go language and still take advantage of the features presented by the native
 KDS Java API libraries.
 
-[vmware-go-kcl](https://github.com/vmware/vmware-go-kcl) is a VMWare originated open-source project for AWS Kinesis 
-Client Library in Go. Within VMware, we have seen adoption in vSecureState and Carbon Black. In addition, Carbon Black 
-has contributed to the vmware-go-kcl codebase and heavily used it in the product. Besides, 
-[vmware-go-kcl](https://github.com/vmware/vmware-go-kcl) has got 
-[recognition](https://www.linkedin.com/posts/adityakrish_vmware-go-kcl-a-native-open-source-go-programming-activity-6810626798133616640-B6W8/), 
-and [contributions](https://github.com/vmware/vmware-go-kcl/graphs/contributors) from the industry.
+[vmware-go-kcl-v2](https://github.com/vmware/vmware-go-kcl-v2) is a VMWare originated open-source project for AWS Kinesis
+Client Library in Go. Within VMware, we have seen adoption in vSecureState and Carbon Black. In addition, Carbon Black
+has contributed to the vmware-go-kcl codebase and heavily used it in the product. Besides,
+[vmware-go-kcl-v2](https://github.com/vmware/vmware-go-kcl-v2) has got
+[recognition](https://www.linkedin.com/posts/adityakrish_vmware-go-kcl-a-native-open-source-go-programming-activity-6810626798133616640-B6W8/),
+and [contributions](https://github.com/vmware/vmware-go-kcl-v2/graphs/contributors) from the industry.
 
 `vmware-go-kcl-v2` is the v2 version of VMWare KCL for the Go programming language by utilizing [AWS Go SDK V2](https://github.com/aws/aws-sdk-go-v2).
 
@@ -31,21 +31,20 @@ and [contributions](https://github.com/vmware/vmware-go-kcl/graphs/contributors)
 ### Build & Run
 
 1. Initialize Project
-2. Build
 
-`make build`
+2. Build
+    > `make build`
 
 3. Test
-
-`make test`
+    > `make test`
 
 ## Documentation
 
 VMware-Go-KCL matches exactly the same interface and programming model from original Amazon KCL, the best place for getting reference, tutorial is from Amazon itself:
 
-- [Developing Consumers Using the Kinesis Client Library](https://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-kcl.html)
-- [Troubleshooting](https://docs.aws.amazon.com/streams/latest/dev/troubleshooting-consumers.html)
-- [Advanced Topics](https://docs.aws.amazon.com/streams/latest/dev/advanced-consumers.html)
+* [Developing Consumers Using the Kinesis Client Library](https://docs.aws.amazon.com/streams/latest/dev/developing-consumers-with-kcl.html)
+* [Troubleshooting](https://docs.aws.amazon.com/streams/latest/dev/troubleshooting-consumers.html)
+* [Advanced Topics](https://docs.aws.amazon.com/streams/latest/dev/advanced-consumers.html)
 
 ## Contributing
 

_support/scripts/ci.sh

@@ -101,7 +101,7 @@ function scanast() {
     rm -f security.log
 }
 
-function Scan() {
+function scan() {
     gosec -fmt=sarif -out=results.sarif -exclude-dir=internal -exclude-dir=vendor -severity=high ./...
 }
 
@@ -117,11 +117,11 @@ function usage() {
 }
 
 case "$1" in
-    fmtcheck) checkfmt ;;
+    fmtCheck) checkfmt ;;
     format) goFormat ;;
     lint) lint ;;
     lintDocker) lintDocker ;;
-    unittest) unitTest ;;
+    unitTest) unitTest ;;
     scan) scan ;;
     localScan) localScan ;;
     *) usage ;;

clientlibrary/checkpoint/dynamodb-checkpointer.go

@@ -90,7 +90,7 @@ func (checkpointer *DynamoCheckpoint) Init() error {
 	checkpointer.log.Infof("Creating DynamoDB session")
 
 	if checkpointer.svc == nil {
-		er := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
+		resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
 			if service == dynamodb.ServiceID && len(checkpointer.kclConfig.DynamoDBEndpoint) > 0 {
 				return aws.Endpoint{
 					PartitionID:   "aws",
@@ -106,7 +106,7 @@ func (checkpointer *DynamoCheckpoint) Init() error {
 			context.TODO(),
 			awsConfig.WithRegion(checkpointer.kclConfig.RegionName),
 			awsConfig.WithCredentialsProvider(checkpointer.kclConfig.DynamoDBCredentials),
-			awsConfig.WithEndpointResolver(er),
+			awsConfig.WithEndpointResolverWithOptions(resolver),
 			awsConfig.WithRetryer(func() aws.Retryer {
 				return retry.AddWithMaxBackoffDelay(retry.NewStandard(), retry.DefaultMaxBackoff)
 			}),

clientlibrary/config/config.go

@@ -43,7 +43,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
 
 	"github.com/vmware/vmware-go-kcl-v2/clientlibrary/metrics"
 	"github.com/vmware/vmware-go-kcl-v2/logger"
@@ -174,10 +173,10 @@ type (
 		KinesisEndpoint string
 
 		// KinesisCredentials is used to access Kinesis
-		KinesisCredentials *credentials.StaticCredentialsProvider
+		KinesisCredentials aws.CredentialsProvider
 
 		// DynamoDBCredentials is used to access DynamoDB
-		DynamoDBCredentials *credentials.StaticCredentialsProvider
+		DynamoDBCredentials aws.CredentialsProvider
 
 		// TableName is name of the dynamo db table for managing kinesis stream default to ApplicationName
 		TableName string

clientlibrary/config/kcl-config.go

@@ -40,7 +40,7 @@ import (
 	"log"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/aws"
 
 	"github.com/vmware/vmware-go-kcl-v2/clientlibrary/metrics"
 	"github.com/vmware/vmware-go-kcl-v2/clientlibrary/utils"
@@ -55,13 +55,13 @@ func NewKinesisClientLibConfig(applicationName, streamName, regionName, workerID
 
 // NewKinesisClientLibConfigWithCredential creates a default KinesisClientLibConfiguration based on the required fields and unique credentials.
 func NewKinesisClientLibConfigWithCredential(applicationName, streamName, regionName, workerID string,
-	creds *credentials.StaticCredentialsProvider) *KinesisClientLibConfiguration {
+	creds aws.CredentialsProvider) *KinesisClientLibConfiguration {
 	return NewKinesisClientLibConfigWithCredentials(applicationName, streamName, regionName, workerID, creds, creds)
 }
 
 // NewKinesisClientLibConfigWithCredentials creates a default KinesisClientLibConfiguration based on the required fields and specific credentials for each service.
 func NewKinesisClientLibConfigWithCredentials(applicationName, streamName, regionName, workerID string,
-	kinesisCreds, dynamodbCreds *credentials.StaticCredentialsProvider) *KinesisClientLibConfiguration {
+	kinesisCreds, dynamodbCreds aws.CredentialsProvider) *KinesisClientLibConfiguration {
 	checkIsValueNotEmpty("ApplicationName", applicationName)
 	checkIsValueNotEmpty("StreamName", streamName)
 	checkIsValueNotEmpty("RegionName", regionName)

clientlibrary/metrics/cloudwatch/cloudwatch.go

@@ -35,8 +35,6 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/credentials"
-
 	cwatch "github.com/aws/aws-sdk-go-v2/service/cloudwatch"
 	"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 
@@ -51,7 +49,7 @@ type MonitoringService struct {
 	streamName  string
 	workerID    string
 	region      string
-	credentials *credentials.StaticCredentialsProvider
+	credentials aws.CredentialsProvider
 	logger      logger.Logger
 
 	// control how often to publish to CloudWatch
@@ -76,13 +74,13 @@ type cloudWatchMetrics struct {
 }
 
 // NewMonitoringService returns a Monitoring service publishing metrics to CloudWatch.
-func NewMonitoringService(region string, creds *credentials.StaticCredentialsProvider) *MonitoringService {
+func NewMonitoringService(region string, creds aws.CredentialsProvider) *MonitoringService {
 	return NewMonitoringServiceWithOptions(region, creds, logger.GetDefaultLogger(), DefaultCloudwatchMetricsBufferDuration)
 }
 
 // NewMonitoringServiceWithOptions returns a Monitoring service publishing metrics to
 // CloudWatch with the provided credentials, buffering duration and logger.
-func NewMonitoringServiceWithOptions(region string, creds *credentials.StaticCredentialsProvider, logger logger.Logger, bufferDur time.Duration) *MonitoringService {
+func NewMonitoringServiceWithOptions(region string, creds aws.CredentialsProvider, logger logger.Logger, bufferDur time.Duration) *MonitoringService {
 	return &MonitoringService{
 		region:         region,
 		credentials:    creds,

clientlibrary/worker/common-shard-consumer.go

@@ -27,13 +27,13 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/kinesis"
 	"github.com/aws/aws-sdk-go-v2/service/kinesis/types"
+	deagg "github.com/awslabs/kinesis-aggregation/go/v2/deaggregator"
 
 	chk "github.com/vmware/vmware-go-kcl-v2/clientlibrary/checkpoint"
 	"github.com/vmware/vmware-go-kcl-v2/clientlibrary/config"
 	kcl "github.com/vmware/vmware-go-kcl-v2/clientlibrary/interfaces"
 	"github.com/vmware/vmware-go-kcl-v2/clientlibrary/metrics"
 	par "github.com/vmware/vmware-go-kcl-v2/clientlibrary/partition"
-	deagg "github.com/vmware/vmware-go-kcl-v2/internal/deaggregator"
 )
 
 type shardConsumer interface {

clientlibrary/worker/worker.go

@@ -40,7 +40,6 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/aws/retry"
 	awsConfig "github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/service/kinesis"
 
 	chk "github.com/vmware/vmware-go-kcl-v2/clientlibrary/checkpoint"
@@ -160,7 +159,7 @@ func (w *Worker) initialize() error {
 		// create session for Kinesis
 		log.Infof("Creating Kinesis client")
 
-		resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
+		resolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
 			return aws.Endpoint{
 				PartitionID:   "aws",
 				URL:           w.kclConfig.KinesisEndpoint,
@@ -171,12 +170,8 @@ func (w *Worker) initialize() error {
 		cfg, err := awsConfig.LoadDefaultConfig(
 			context.TODO(),
 			awsConfig.WithRegion(w.regionName),
-			awsConfig.WithCredentialsProvider(
-				credentials.NewStaticCredentialsProvider(
-					w.kclConfig.KinesisCredentials.Value.AccessKeyID,
-					w.kclConfig.KinesisCredentials.Value.SecretAccessKey,
-					w.kclConfig.KinesisCredentials.Value.SessionToken)),
-			awsConfig.WithEndpointResolver(resolver),
+			awsConfig.WithCredentialsProvider(w.kclConfig.KinesisCredentials),
+			awsConfig.WithEndpointResolverWithOptions(resolver),
 			awsConfig.WithRetryer(func() aws.Retryer {
 				return retry.AddWithMaxBackoffDelay(retry.NewStandard(), retry.DefaultMaxBackoff)
 			}),

go.mod

@@ -3,35 +3,36 @@ module github.com/vmware/vmware-go-kcl-v2
 go 1.17
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.11.0
-	github.com/aws/aws-sdk-go-v2/config v1.10.0
-	github.com/aws/aws-sdk-go-v2/credentials v1.6.0
-	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.10.0
-	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.7.0
-	github.com/aws/aws-sdk-go-v2/service/kinesis v1.8.0
+	github.com/aws/aws-sdk-go-v2 v1.11.2
+	github.com/aws/aws-sdk-go-v2/config v1.11.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.6.5
+	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.13.0
+	github.com/aws/aws-sdk-go-v2/service/dynamodb v1.11.0
+	github.com/aws/aws-sdk-go-v2/service/kinesis v1.11.0
+	github.com/awslabs/kinesis-aggregation/go/v2 v2.0.0-20211222152315-953b66f67407
 	github.com/golang/protobuf v1.5.2
 	github.com/google/uuid v1.3.0
 	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/common v0.32.1
-	github.com/rs/zerolog v1.26.0
+	github.com/rs/zerolog v1.26.1
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
-	go.uber.org/zap v1.19.1
+	go.uber.org/zap v1.20.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (
 	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.8.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.0.2 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.5.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.3.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.6.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.9.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.3.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.5.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.7.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.12.0 // indirect
 	github.com/aws/smithy-go v1.9.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -43,7 +44,7 @@ require (
 	github.com/prometheus/procfs v0.7.3 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
-	golang.org/x/sys v0.0.0-20211111213525-f221eed1c01e // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )