AWS IoT SDK v2 does not work in lambda functions

[Th3G4mbl3r]

Confirm by changing [ ] to [x] below to ensure that it's a bug:

• I've searched for previous similar issues and didn't find any solution

Describe the bug
AWS IoT SDK depends on aws-crt. As soon as i try to import any object from aws-crt the lambda function with Runtime.UserCodeSyntaxError.

SDK version number

"aws-crt": "^1.1.5",
"aws-iot-device-sdk-v2": "^1.2.1"

Platform/OS/Hardware/Device
MacOS Catalina 10.15.5 and node 10.21.0

To Reproduce (observed behavior)
Steps to reproduce the behavior (please share code)

import {io, mqtt, iot} from "aws-crt";

let amazon_ca_cert = process.env.amazon_ca_cert;
let device_cert_file = process.env.device_cert_file;
let device_key_file = process.env.device_key_file;
let endpoint = process.env.endpoint;
let valve_name_suffix = process.env.valve_name_suffix;
let certs_directory = process.env.certs_directory;

const isUndefined = require('./isundefined');

var valveShadow = null;

var msgMotorStateOn = {
    "state": {
        "desired": {
            "motor_state": "on"
        }
    }
};

async function valveConnect(valve_name, cert_name, key_name)
{
    console.debug('Inside valveConnect');
    //const client_bootstrap = new io.ClientBootstrap();

    let config_builder = null;

    console.debug('creating Config Builder object...');
    //config_builder = iot.AwsIotMqttConnectionConfigBuilder.new_mtls_builder_from_path(cert_name, key_name);
    //config_builder.with_clean_session(false);
    //config_builder.with_client_id(valve_name);
    //config_builder.with_endpoint(endpoint);

    console.debug('Creating a timer to keep the lambda function running in the background while processing is happening for the connection.');
    const timer = setTimeout(() => {}, 60 * 1000);  //Forcing nodejs function to wait for 60 seconds before the entire function times out as SDK is promise based.

    //const config = config_builder.build();

    console.debug('Obtaining a new MQTTClient object.');
    //const client = new mqtt.MqttClient(client_bootstrap);

    console.debug('Obtaining a new MQTT Client Connection object.');
    //const connection = client.new_connection(config);

    console.debug('Starting the connection process and waiting till it finishes completely...');
    //await connection.connect().promise();

    // Allow node to die if the promise above resolved
    clearTimeout(timer);
}

function activateValveMotor(valve_name)
{
    console.debug('Inside activateValveMotor');

    console.debug('Obtaining shadow object for Valve: ' + valve_name);

    let cert_name = certs_directory + valve_name + device_cert_file;
    let key_name = certs_directory + valve_name + device_key_file;

    console.debug('Initiating connection for valve  ' + valve_name + ' with key ' + key_name + ' and certificate ' + cert_name);

    valveConnect(valve_name, cert_name, key_name).then(() => console.debug('Successfully connected.')).catch();
}

exports.iotMB3ValveActuatorHandler = async (event, context) => {
    // All log statements are written to CloudWatch by default. For more information, see
    // https://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-logging.html

    console.debug('Starting the lambda processing in response to motor request...');
    console.debug('event value is: ' + event);

    console.debug('Value of amazon_ca_cert = ' + amazon_ca_cert);
    console.debug('Value of device_cert_file = ' + device_cert_file);
    console.debug('Value of device_key_file = ' + device_key_file);
    console.debug('Value of endpoint = ' + endpoint);
    console.debug('Value of valve_name_suffix = ' + valve_name_suffix);
    console.debug('Value of certs_directory = ' + certs_directory);

    let intent_name = event['currentIntent']['name'];
    let slots = event['currentIntent']['slots'];
    let valve_name = valve_name_suffix + slots['valve_name'];

    console.debug('Value of intent_name = ' + intent_name);
    console.debug('Value of slots = ' + slots);
    console.debug('Value of valve_name = ' + valve_name);

    //console.debug('Setting up SDK Log Level = ' + io.LogLevel.DEBUG);
    //io.enable_logging(io.LogLevel.DEBUG);

    if (intent_name == 'ActivateValve')
    {
        console.debug('Received an Activate Valve request from Lex...');
        activateValveMotor(valve_name);
        setTimeout(function(){console.debug('Waiting for valve activation to complete in 15 seconds.')}, 15000);
    }
}

Expected behavior
Lambda function should be able to load completely and start excuting.

Logs/output

{
  "errorType": "Runtime.UserCodeSyntaxError",
  "errorMessage": "SyntaxError: Unexpected token {",
  "trace": [
    "Runtime.UserCodeSyntaxError: SyntaxError: Unexpected token {",
    "    at _loadUserApp (/var/runtime/UserFunction.js:98:13)",
    "    at Object.module.exports.load (/var/runtime/UserFunction.js:140:17)",
    "    at Object.<anonymous> (/var/runtime/index.js:45:30)",
    "    at Module._compile (internal/modules/cjs/loader.js:778:30)",
    "    at Object.Module._extensions..js (internal/modules/cjs/loader.js:789:10)",
    "    at Module.load (internal/modules/cjs/loader.js:653:32)",
    "    at tryModuleLoad (internal/modules/cjs/loader.js:593:12)",
    "    at Function.Module._load (internal/modules/cjs/loader.js:585:3)",
    "    at Function.Module.runMain (internal/modules/cjs/loader.js:831:12)",
    "    at startup (internal/bootstrap/node.js:283:19)"
  ]
}

[bretambrose]

Documenting a few things here, no official diagnosis yet:

ldd -r -v aws-crt-nodejs.node

linux-vdso.so.1 (0x00007ffc01bd9000)
librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007f3b0aa30000)
libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f3b0a692000)
libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f3b0a48e000)
libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007f3b0a26f000)
libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f3b09e7e000)
/lib64/ld-linux-x86-64.so.2 (0x00007f3b0b251000)

undefined symbol: napi_coerce_to_bool (./aws-crt-nodejs.node)
(snip)

Version information:
./aws-crt-nodejs.node:
	librt.so.1 (GLIBC_2.2.5) => /lib/x86_64-linux-gnu/librt.so.1
	libdl.so.2 (GLIBC_2.2.5) => /lib/x86_64-linux-gnu/libdl.so.2
	ld-linux-x86-64.so.2 (GLIBC_2.3) => /lib64/ld-linux-x86-64.so.2
	libpthread.so.0 (GLIBC_2.3.2) => /lib/x86_64-linux-gnu/libpthread.so.0
	libpthread.so.0 (GLIBC_2.2.5) => /lib/x86_64-linux-gnu/libpthread.so.0
	libc.so.6 (GLIBC_2.14) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.3) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.25) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.17) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.7) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.3.2) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.3.4) => /lib/x86_64-linux-gnu/libc.so.6
	libc.so.6 (GLIBC_2.2.5) => /lib/x86_64-linux-gnu/libc.so.6
/lib/x86_64-linux-gnu/librt.so.1:

(snip)

Of particular interest is the GLIBC_2.25 line. The lambda sandbox's glibc was not that high as of a few years ago, unknown what it is presently, but this may indicate that the native parts of the crt are built against a later glibc than what lambda supports.

[bretambrose]

This blog post

https://cloudbriefly.com/post/exploring-the-aws-lambda-execution-environment/

seems to indicate that one of the images used as a lambda sandbox has glibc 2.17, while the other uses 2.26. Based on this documentation

https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html

it appears that all versions of node are running on AL2 which should have 2.26, so this may be a dead end.

[Th3G4mbl3r]

This blog post

https://cloudbriefly.com/post/exploring-the-aws-lambda-execution-environment/

seems to indicate that one of the images used as a lambda sandbox has glibc 2.17, while the other uses 2.26. Based on this documentation

https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html

it appears that all versions of node are running on AL2 which should have 2.26, so this may be a dead end.

so what does this mean? Does it mean that the new version of the sdk cannot be used in lambda functions?

[bretambrose]

Not able to conclude anything yet. I'm just documenting what I've looked into so far. If you are able to open an internal ticket that would be ideal.

[Th3G4mbl3r]

Not able to conclude anything yet. I'm just documenting what I've looked into so far. If you are able to open an internal ticket that would be ideal.

Hi Bret,

Thanks for the quick response. I've reached out to you internally as well on the internal ticket process. In the meantime i will leave this open in case someone else faces the same issue.

thanks & regards,
rohit

[Th3G4mbl3r]

based on feedback on internal AWS ticket, realized that instead of using the "import" statement approach, using "require" works. So i will close this issue for now. The existing sample codes should be modified to reflect the same as well.