diff --git a/.env b/.env index 52b3b94..a926869 100644 --- a/.env +++ b/.env @@ -1,3 +1,3 @@ -SYSROOT_VERSION=0.5.0 -SYSROOT_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.5.0 +SYSROOT_VERSION=0.6.0 +SYSROOT_CLI_IMAGE=faasm.azurecr.io/cpp-sysroot:0.6.0 COMPOSE_PROJECT_NAME=cpp-dev diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc0ae58..8c60ba0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -17,7 +17,7 @@ jobs: if: github.event.pull_request.draft == false runs-on: ubuntu-latest container: - image: faasm.azurecr.io/cpp-sysroot:0.5.0 + image: faasm.azurecr.io/cpp-sysroot:0.6.0 steps: # --- Update code --- - name: "Checkout code" diff --git a/VERSION b/VERSION index 8f0916f..a918a2a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.0 +0.6.0 diff --git a/func/CMakeLists.txt b/func/CMakeLists.txt index 47f4f86..2a02fd8 100644 --- a/func/CMakeLists.txt +++ b/func/CMakeLists.txt @@ -74,4 +74,5 @@ add_subdirectory(dynlink) add_subdirectory(errors) add_subdirectory(mpi) add_subdirectory(omp) +add_subdirectory(s3) add_subdirectory(threads) diff --git a/func/demo/chain_output.cpp b/func/demo/chain_output.cpp index 8839b5a..2b8e524 100644 --- a/func/demo/chain_output.cpp +++ b/func/demo/chain_output.cpp @@ -26,17 +26,22 @@ int main(int argc, char* argv[]) unsigned int callIdB = faasmChain(otherB, nullptr, 0); std::string expectedA = "expected A"; - std::string actualA; - actualA.reserve(expectedA.size()); std::string expectedB = "longer expected B"; - std::string actualB; - actualB.reserve(expectedB.size()); + std::string actualA; + char* actualABuf; + int actualABufSize; unsigned int resA = - faasmAwaitCallOutput(callIdA, actualA.c_str(), actualA.size()); + faasmAwaitCallOutput(callIdA, &actualABuf, &actualABufSize); + actualA.assign(actualABuf, actualABuf + actualABufSize); + + std::string actualB; + char* actualBBuf; + int actualBBufSize; unsigned int resB = - faasmAwaitCallOutput(callIdB, actualB.c_str(), actualB.size()); + faasmAwaitCallOutput(callIdB, &actualBBuf, &actualBBufSize); + actualB.assign(actualBBuf, actualBBuf + actualBBufSize); if (resA != 0 || resB != 0) { printf("One or more chained calls failed: %i %i\n", resA, resB); @@ -44,10 +49,14 @@ int main(int argc, char* argv[]) } if (actualA != expectedA) { + printf( + "Output mismatch: %s != %s\n", actualA.c_str(), expectedA.c_str()); return 1; } if (actualB != expectedB) { + printf( + "Output mismatch: %s != %s\n", actualB.c_str(), expectedB.c_str()); return 1; } diff --git a/func/s3/CMakeLists.txt b/func/s3/CMakeLists.txt new file mode 100644 index 0000000..a082f11 --- /dev/null +++ b/func/s3/CMakeLists.txt @@ -0,0 +1,16 @@ +set(FAASM_USER s3) + +function(s3_func exec_name dir_path) + faasm_func(${exec_name} ${dir_path}) + set(ALL_DEMO_FUNCS ${ALL_DEMO_FUNCS} ${exec_name} PARENT_SCOPE) +endfunction(s3_func) + +s3_func(get_num_buckets get_num_buckets.cpp) +s3_func(list_buckets list_buckets.cpp) +s3_func(get_num_keys get_num_keys.cpp) +s3_func(list_keys list_keys.cpp) +s3_func(add_key_bytes add_key_bytes.cpp) +s3_func(get_key_bytes get_key_bytes.cpp) + +# Custom target to group all the demo functions +add_custom_target(s3_all_funcs DEPENDS ${ALL_DEMO_FUNCS}) diff --git a/func/s3/add_key_bytes.cpp b/func/s3/add_key_bytes.cpp new file mode 100644 index 0000000..67708c7 --- /dev/null +++ b/func/s3/add_key_bytes.cpp @@ -0,0 +1,30 @@ +extern "C" +{ +#include "faasm/host_interface.h" +} + +#include +#include + +int main(int argc, char* argv[]) +{ + // Get bucket and key from command line + if (argc != 3) { + printf("error: must invoke function with two arguments: bucketName " + "keyName\n"); + return 1; + } + + char* bucketName = argv[1]; + char* keyName = argv[2]; + + // Get the bytes to add as input + int inputSize = faasmGetInputSize(); + uint8_t keyBytes[inputSize]; + faasmGetInput(keyBytes, inputSize); + + int ret = + __faasm_s3_add_key_bytes(bucketName, keyName, (void*)keyBytes, inputSize); + + return ret; +} diff --git a/func/s3/get_key_bytes.cpp b/func/s3/get_key_bytes.cpp new file mode 100644 index 0000000..29c0ecf --- /dev/null +++ b/func/s3/get_key_bytes.cpp @@ -0,0 +1,30 @@ +extern "C" +{ +#include "faasm/host_interface.h" +} + +#include +#include + +int main(int argc, char* argv[]) +{ + // Get bucket and key from command line + if (argc != 3) { + printf("error: must invoke function with two arguments: bucketName " + "keyName\n"); + return 1; + } + + char* bucketName = argv[1]; + char* keyName = argv[2]; + + uint8_t* keyBytes; + int keyBytesLen; + + int ret = + __faasm_s3_get_key_bytes(bucketName, keyName, &keyBytes, &keyBytesLen); + printf("Got %s/%s: %s\n", bucketName, keyName, (char*)keyBytes); + faasmSetOutput((char*)keyBytes, keyBytesLen); + + return ret; +} diff --git a/func/s3/get_num_buckets.cpp b/func/s3/get_num_buckets.cpp new file mode 100644 index 0000000..b092767 --- /dev/null +++ b/func/s3/get_num_buckets.cpp @@ -0,0 +1,20 @@ +extern "C" +{ +#include "faasm/host_interface.h" +} + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + int numBuckets = __faasm_s3_get_num_buckets(); + + printf("Got %i buckets!\n", numBuckets); + + std::string numBucketsStr = std::to_string(numBuckets); + faasmSetOutput(numBucketsStr.c_str(), numBucketsStr.size()); + + return 0; +} diff --git a/func/s3/get_num_keys.cpp b/func/s3/get_num_keys.cpp new file mode 100644 index 0000000..8f9ed92 --- /dev/null +++ b/func/s3/get_num_keys.cpp @@ -0,0 +1,25 @@ +extern "C" +{ +#include "faasm/host_interface.h" +} + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + // Get the bucket name as an input + int inputSize = faasmGetInputSize(); + char bucketName[inputSize]; + faasmGetInput((uint8_t*)bucketName, inputSize); + + int numKeys = __faasm_s3_get_num_keys(bucketName); + + printf("Bucket %s has %i keys!\n", bucketName, numKeys); + + std::string numKeysStr = std::to_string(numKeys); + faasmSetOutput(numKeysStr.c_str(), numKeysStr.size()); + + return 0; +} diff --git a/func/s3/list_buckets.cpp b/func/s3/list_buckets.cpp new file mode 100644 index 0000000..46a1787 --- /dev/null +++ b/func/s3/list_buckets.cpp @@ -0,0 +1,42 @@ +extern "C" +{ +#include "faasm/host_interface.h" +} + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + int numBuckets = __faasm_s3_get_num_buckets(); + + char* bucketsBuffer[numBuckets]; + int bucketsBufferLens[numBuckets]; + __faasm_s3_list_buckets(bucketsBuffer, bucketsBufferLens); + + int totalSize = 0; + for (int i = 0; i < numBuckets; i++) { + totalSize += bucketsBufferLens[i]; + } + totalSize += numBuckets - 1; + + // Prepare the output: instead of a newline use a '|' character + char outBuffer[totalSize]; + + printf("Got %i buckets!\n", numBuckets); + int offset = 0; + for (int i = 0; i < numBuckets; i++) { + strncpy(outBuffer + offset, bucketsBuffer[i], bucketsBufferLens[i]); + offset += bucketsBufferLens[i]; + if (i < numBuckets - 1) { + outBuffer[offset] = (char)'|'; + offset += 1; + } + printf("Bucket %i: %s\n", i, bucketsBuffer[i]); + } + + faasmSetOutput(outBuffer, totalSize); + + return 0; +} diff --git a/func/s3/list_keys.cpp b/func/s3/list_keys.cpp new file mode 100644 index 0000000..ef9b224 --- /dev/null +++ b/func/s3/list_keys.cpp @@ -0,0 +1,47 @@ +extern "C" +{ +#include "faasm/host_interface.h" +} + +#include +#include +#include + +int main(int argc, char* argv[]) +{ + // Get the bucket name as an input + int inputSize = faasmGetInputSize(); + char bucketName[inputSize]; + faasmGetInput((uint8_t*)bucketName, inputSize); + + int numKeys = __faasm_s3_get_num_keys(bucketName); + + char* keysBuffer[numKeys]; + int keysBufferLens[numKeys]; + __faasm_s3_list_keys(bucketName, keysBuffer, keysBufferLens); + + int totalSize = 0; + for (int i = 0; i < numKeys; i++) { + totalSize += keysBufferLens[i]; + } + totalSize += numKeys - 1; + + // Prepare the output: instead of a newline use a '|' character + char outBuffer[totalSize]; + + printf("Bucket %s has %i keys!\n", bucketName, numKeys); + int offset = 0; + for (int i = 0; i < numKeys; i++) { + strncpy(outBuffer + offset, keysBuffer[i], keysBufferLens[i]); + offset += keysBufferLens[i]; + if (i < numKeys - 1) { + outBuffer[offset] = (char)'|'; + offset += 1; + } + printf("Key %i: %s\n", i, keysBuffer[i]); + } + + faasmSetOutput(outBuffer, totalSize); + + return 0; +} diff --git a/libfaasm/core.cpp b/libfaasm/core.cpp index a7b29bd..0fef908 100644 --- a/libfaasm/core.cpp +++ b/libfaasm/core.cpp @@ -152,8 +152,8 @@ unsigned int faasmAwaitCall(unsigned int messageId) } unsigned int faasmAwaitCallOutput(unsigned int messageId, - const char* output, - long outputLen) + char** output, + int* outputLen) { return __faasm_await_call_output(messageId, output, outputLen); } diff --git a/libfaasm/faasm/core.h b/libfaasm/faasm/core.h index 351b700..bb755eb 100644 --- a/libfaasm/faasm/core.h +++ b/libfaasm/faasm/core.h @@ -188,8 +188,8 @@ extern "C" * Gets the output from the given call into the buffer */ unsigned int faasmAwaitCallOutput(unsigned int messageId, - const char* output, - long outputLen); + char** output, + int* outputLen); /** * Returns the python user diff --git a/libfaasm/faasm/host_interface.h b/libfaasm/faasm/host_interface.h index 00a7573..ba7f4b7 100644 --- a/libfaasm/faasm/host_interface.h +++ b/libfaasm/faasm/host_interface.h @@ -114,8 +114,8 @@ int __faasm_await_call(unsigned int messageId); HOST_IFACE_FUNC int __faasm_await_call_output(unsigned int messageId, - const char* output, - long outputLen); + char** output, + int* outputLen); HOST_IFACE_FUNC void __faasm_get_py_user(unsigned char* buffer, long bufferLen); @@ -160,4 +160,35 @@ void __faasm_migrate_point(FaasmMigrateEntryPoint f, int arg); HOST_IFACE_FUNC void __faasm_host_interface_test(int testNum); + +// ----- S3 ----- + +HOST_IFACE_FUNC +int __faasm_s3_get_num_buckets(); + +// Note that bucketsBuffer is, in reality, a char** populated by the host +HOST_IFACE_FUNC +void __faasm_s3_list_buckets(void* bucketsBuffer, int* bucketsBufferLens); + +HOST_IFACE_FUNC +int __faasm_s3_get_num_keys(const char* bucketName); + +// Note that keysBuffer is, in reality, a char** populated by the host +HOST_IFACE_FUNC +void __faasm_s3_list_keys(const char* bucketName, + void* keysBuffer, + int* keysBufferLens); + +HOST_IFACE_FUNC +int __faasm_s3_add_key_bytes(const char* bucketName, + const char* keyName, + void* keyBuffer, + int keyBufferLen); + +// Note that keyBuffer is, in reality, a uint8_t** populated by the host +HOST_IFACE_FUNC +int __faasm_s3_get_key_bytes(const char* bucketName, + const char* keyName, + void* keyBuffer, + int* keyBufferLen); #endif diff --git a/libfaasm/libfaasm.imports b/libfaasm/libfaasm.imports index 6ca3203..76ff179 100644 --- a/libfaasm/libfaasm.imports +++ b/libfaasm/libfaasm.imports @@ -42,5 +42,13 @@ __faasm_sm_critical_local_end # Migration __faasm_migrate_point +# S3 +__faasm_s3_get_num_buckets +__faasm_s3_list_buckets +__faasm_s3_get_num_keys +__faasm_s3_list_keys +__faasm_s3_add_key_bytes +__faasm_s3_get_key_bytes + # Test __faasm_host_interface_test diff --git a/tasks/func.py b/tasks/func.py index 2461cd6..409d450 100644 --- a/tasks/func.py +++ b/tasks/func.py @@ -179,8 +179,16 @@ def local(ctx, clean=False, debug=False): """ Compile all functions used in the tests """ + # Mixing users that use the threads sysroot and the non-threads sysroot + # seems to give some rather obscure compilation errors, so just make sure + # to separate them + + # Non-threaded users user(ctx, "demo", clean, debug) user(ctx, "errors", clean, debug) user(ctx, "mpi", clean, debug) + user(ctx, "s3", clean, debug) + + # Threaded users user(ctx, "omp", clean, debug) user(ctx, "threads", clean, debug)