Issue 2457 (#2502)

* Os::File refactor: open w/ basic stub, posix UTs

* Refactoring Os::File for all methods, less UTs

* Adding rules-based common tests

* Fixing posix file for common UTs

* Adding WriteRead and WriteReadSeek tests

* Working random tests

* Working randomized tests and stub testing

* Fixing PrmDb unit tests

* Updating files to used FwSignedSizeType

* Factoring out the synthetic file system

* Initial work on CmdSequencer UTs

* Fixing linux implementation

* Newlines at end of file

* sp

* Static analysis checks

* Changing API to use enums

* Refactoring for proper class tree

* CmdSequencer UTs - WIP

* CmdSequencer UTs work

* Adding copy constructor functionality

* Spelling and static analysis fixes

* Copy constructor/assignment operator UTs and fixes

* Initial CRC work

* Working CRC and FPP models

* Minor UT fixes

* Correcting CMake choices implementation

* MAking implementation choices real dependencies

* Fixing implementation to use source files

* Fixing posix file read/write permissions

* Allowing None implementations

* Fixed CI breakages

* Fixing MyRules.hpp formatting

* Minor CRC unit test fix

* Removing direct write support

* Changing get delegate to static method of interface

* Replacing old comments with new comments

* Fixing friend regression

* Review fixes

* Fixing serialization method usages

* Review fixes

* Fixing RPI build

* Fixing FwBuffSizeType overflow

* Fixing review comment

* Renaming FwExternalSize -> FwSizeStoreType

Fw/Cfg/ConfigCheck.cpp

@@ -11,6 +11,7 @@
  */
 
 #include <FpConfig.hpp>
+#include <limits>
 
 // Check that command/telemetry strings are not larger than an argument buffer
 
@@ -23,3 +24,10 @@ static_assert(FW_PARAM_STRING_MAX_SIZE <= FW_PARAM_BUFFER_MAX_SIZE, "FW_PARAM_ST
 // value.
 static_assert((FW_ENABLE_TEXT_LOGGING == 0) || ( FW_SERIALIZABLE_TO_STRING == 1), "FW_SERIALIZABLE_TO_STRING must be enabled to enable FW_ENABLE_TEXT_LOGGING");
 
+static_assert(std::numeric_limits<FwBuffSizeType>::max() == std::numeric_limits<FwSizeStoreType>::max() &&
+              std::numeric_limits<FwBuffSizeType>::min() == std::numeric_limits<FwSizeStoreType>::min(),
+              "FwBuffSizeType must be equivalent to FwExternalSizeType");
+
+static_assert(std::numeric_limits<FwSizeType>::max() >= std::numeric_limits<FwSizeStoreType>::max() &&
+              std::numeric_limits<FwSizeType>::min() <= std::numeric_limits<FwSizeStoreType>::min(),
+              "FwSizeType cannot entirely store values of type FwExternalSizeType");

Fw/Dp/DpContainer.cpp

@@ -70,10 +70,9 @@ Fw::SerializeStatus DpContainer::deserializeHeader() {
     }
     // Deserialize the user data
     if (status == Fw::FW_SERIALIZE_OK) {
-        const bool omitLength = true;
         const FwSizeType requestedSize = sizeof this->m_userData;
         FwSizeType receivedSize = requestedSize;
-        status = serializeRepr.deserialize(this->m_userData, receivedSize, omitLength);
+        status = serializeRepr.deserialize(this->m_userData, receivedSize, Fw::Serialization::OMIT_LENGTH);
         if (receivedSize != requestedSize) {
             status = Fw::FW_DESERIALIZE_SIZE_MISMATCH;
         }
@@ -111,8 +110,8 @@ void DpContainer::serializeHeader() {
     status = serializeRepr.serialize(this->m_procTypes);
     FW_ASSERT(status == Fw::FW_SERIALIZE_OK, static_cast<FwAssertArgType>(status));
     // Serialize the user data
-    const bool omitLength = true;
-    status = serializeRepr.serialize(this->m_userData, sizeof this->m_userData, omitLength);
+    status = serializeRepr.serialize(this->m_userData, static_cast<FwSizeType>(sizeof this->m_userData),
+                                     Fw::Serialization::OMIT_LENGTH);
     FW_ASSERT(status == Fw::FW_SERIALIZE_OK, static_cast<FwAssertArgType>(status));
     // Serialize the data product state
     status = serializeRepr.serialize(this->m_dpState);

Fw/SerializableFile/CMakeLists.txt

@@ -12,13 +12,10 @@ set(SOURCE_FILES
   "${CMAKE_CURRENT_LIST_DIR}/SerializableFile.cpp"
 )
 set(MOD_DEPS
-  Fw/Cfg
-  Fw/Types
+    Fw/Cfg
+    Fw/Types
+    Os
 )
-# For shared libraries, Os must exist in the module list
-if (BUILD_SHARED_LIBS)
-    list(APPEND MOD_DEPS "Os")
-endif()
 register_fprime_module()
 ### UTs ###
 set(UT_SOURCE_FILES

Fw/SerializableFile/SerializableFile.cpp

@@ -39,9 +39,9 @@ namespace Fw {
       return FILE_OPEN_ERROR;
     }
 
-    NATIVE_INT_TYPE capacity = this->m_buffer.getBuffCapacity();
-    NATIVE_INT_TYPE length = capacity;
-    status = file.read(this->m_buffer.getBuffAddr(), length, false);
+    FwSignedSizeType capacity = static_cast<FwSignedSizeType>(this->m_buffer.getBuffCapacity());
+    FwSignedSizeType length = static_cast<FwSignedSizeType>(capacity);
+    status = file.read(this->m_buffer.getBuffAddr(), length, Os::File::WaitType::NO_WAIT);
     if( Os::File::OP_OK != status ) {
       file.close();
       return FILE_READ_ERROR;
@@ -72,12 +72,10 @@ namespace Fw {
       return FILE_OPEN_ERROR;
     }
 
-    NATIVE_INT_TYPE length = this->m_buffer.getBuffLength();
-    NATIVE_INT_TYPE size = length;
+    FwSignedSizeType length = static_cast<FwSignedSizeType>(this->m_buffer.getBuffLength());
+    FwSignedSizeType size = length;
     status = file.write(this->m_buffer.getBuffAddr(), length);
-    if( (Os::File::OP_OK != status) ||
-        (length != size) )
-    {
+    if( (Os::File::OP_OK != status) || (length != size)) {
       file.close();
       return FILE_WRITE_ERROR;
     }

Fw/Types/Serializable.cpp

@@ -234,10 +234,18 @@ namespace Fw {
 
     }
 
+    SerializeStatus SerializeBufferBase::serialize(const U8* buff, NATIVE_UINT_TYPE length) {
+        return this->serialize(buff, static_cast<FwSizeType>(length), Serialization::INCLUDE_LENGTH);
+    }
+
     SerializeStatus SerializeBufferBase::serialize(const U8* buff, NATIVE_UINT_TYPE length, bool noLength) {
+        return this->serialize(buff, static_cast<FwSizeType>(length), noLength ? Serialization::OMIT_LENGTH : Serialization::INCLUDE_LENGTH);
+    }
+
+    SerializeStatus SerializeBufferBase::serialize(const U8* buff, FwSizeType length, Fw::Serialization::t mode) {
         // First serialize length
         SerializeStatus stat;
-        if (not noLength) {
+        if (mode == Serialization::INCLUDE_LENGTH) {
             stat = this->serialize(static_cast<FwBuffSizeType>(length));
             if (stat != FW_SERIALIZE_OK) {
                 return stat;
@@ -499,11 +507,25 @@ namespace Fw {
         return FW_SERIALIZE_OK;
     }
 
+    SerializeStatus SerializeBufferBase::deserialize(U8* buff, NATIVE_UINT_TYPE& length) {
+        FwSizeType length_in_out = static_cast<FwSizeType>(length);
+        SerializeStatus status = this->deserialize(buff, length_in_out, Serialization::INCLUDE_LENGTH);
+        length = static_cast<NATIVE_UINT_TYPE>(length_in_out);
+        return status;
+    }
+
     SerializeStatus SerializeBufferBase::deserialize(U8* buff, NATIVE_UINT_TYPE& length, bool noLength) {
+        FwSizeType length_in_out = static_cast<FwSizeType>(length);
+        SerializeStatus status = this->deserialize(buff, length_in_out, noLength ? Serialization::OMIT_LENGTH : Serialization::INCLUDE_LENGTH);
+        length = static_cast<NATIVE_UINT_TYPE>(length_in_out);
+        return status;
+    }
+
+    SerializeStatus SerializeBufferBase::deserialize(U8* buff, FwSizeType& length, Serialization::t mode) {
 
         FW_ASSERT(this->getBuffAddr());
 
-        if (not noLength) {
+        if (mode == Serialization::INCLUDE_LENGTH) {
             FwBuffSizeType storedLength;
 
             SerializeStatus stat = this->deserialize(storedLength);
@@ -519,7 +541,7 @@ namespace Fw {
 
             (void) memcpy(buff, &this->getBuffAddr()[this->m_deserLoc], storedLength);
 
-            length = static_cast<NATIVE_UINT_TYPE>(storedLength);
+            length = static_cast<FwSizeType>(storedLength);
 
         } else {
             // make sure enough is left

Fw/Types/Serializable.hpp

@@ -6,6 +6,7 @@
 #endif
 
 #include <FpConfig.hpp>
+#include "Fw/Deprecate.hpp"
 
 namespace Fw {
 
@@ -38,6 +39,14 @@ namespace Fw {
             virtual ~Serializable(); //!< destructor
     };
 
+    class Serialization {
+      public:
+        enum t {
+            INCLUDE_LENGTH, //!< Include length as first token in serialization
+            OMIT_LENGTH //!< Omit length from serialization
+        };
+    };
+
     class SerializeBufferBase {
         public:
 
@@ -70,7 +79,21 @@ namespace Fw {
 
             SerializeStatus serialize(const void* val); //!< serialize pointer (careful, only pointer value, not contents are serialized)
 
-            SerializeStatus serialize(const U8* buff, NATIVE_UINT_TYPE length, bool noLength = false); //!< serialize data buffer
+            //! serialize data buffer
+            SerializeStatus serialize(const U8* buff, NATIVE_UINT_TYPE length, bool noLength);
+            //! serialize data buffer
+            SerializeStatus serialize(const U8* buff, NATIVE_UINT_TYPE length);
+
+            //! \brief serialize a byte buffer of a given length
+            //!
+            //! Serialize bytes from `buff` up to `length`.  If `serializationMode` is set to `INCLUDE_LENGTH` then the
+            //! length is included as the first token. Length may be omitted with `OMIT_LENGTH`.
+            //!
+            //! \param buff: buffer to serialize
+            //! \param length: length of data to serialize
+            //! \param mode: serialization type
+            //! \return status of serialization
+            SerializeStatus serialize(const U8* buff, FwSizeType length, Serialization::t mode);
 
             SerializeStatus serialize(const SerializeBufferBase& val); //!< serialize a serialized buffer
 
@@ -102,11 +125,23 @@ namespace Fw {
 
             SerializeStatus deserialize(void*& val); //!< deserialize point value (careful, pointer value only, not contents)
 
-            // length should be set to max, returned value is actual size stored. If noLength
-            // is true, use the length variable as the actual number of bytes to deserialize
-            SerializeStatus deserialize(U8* buff, NATIVE_UINT_TYPE& length, bool noLength = false); //!< deserialize data buffer
-            // serialize/deserialize Serializable
-
+            //! deserialize data buffer
+            SerializeStatus deserialize(U8* buff, NATIVE_UINT_TYPE& length, bool noLength);
+
+            //! deserialize data buffer
+            SerializeStatus deserialize(U8* buff, NATIVE_UINT_TYPE& length);
+            //! \brief deserialize a byte buffer of a given length
+            //!
+            //! Deserialize bytes into `buff` of `length` bytes.  If `serializationMode` is set to `INCLUDE_LENGTH` then
+            //! the length is deserialized first followed by the bytes. Length may be omitted with `OMIT_LENGTH` and
+            //! in this case `length` bytes will be deserialized. `length` will be filled with the amount of data
+            //! deserialized.
+            //!
+            //! \param buff: buffer to hold deserialized data
+            //! \param length: length of data to deserialize length is filled with deserialized length
+            //! \param mode: deserialization type
+            //! \return status of serialization
+            SerializeStatus deserialize(U8* buff, FwSizeType& length, Serialization::t mode);
 
             SerializeStatus deserialize(Serializable &val);  //!< deserialize an object derived from serializable base class
 

Os/CMakeLists.txt

@@ -5,6 +5,15 @@
 # MOD_DEPS: (optional) module dependencies
 #
 ####
+add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/Models")
+add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/Stub")
+add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/Posix")
+
+set(SOURCE_FILES
+    "${CMAKE_CURRENT_LIST_DIR}/test/ut/file/SyntheticFileSystem.cpp"
+)
+register_fprime_module(Os_Test_File_SyntheticFileSystem)
+
 
 # Basic module dependencies
 set(MOD_DEPS
@@ -26,7 +35,9 @@ set(SOURCE_FILES
     "${CMAKE_CURRENT_LIST_DIR}/MemCommon.cpp"
     "${CMAKE_CURRENT_LIST_DIR}/ValidateFileCommon.cpp"
     "${CMAKE_CURRENT_LIST_DIR}/ValidatedFile.cpp"
-    "${CMAKE_CURRENT_LIST_DIR}/FileCommon.cpp"
+
+    # Refactored common files
+    "${CMAKE_CURRENT_LIST_DIR}/File.cpp"
 )
 # Check for default logger
 if (NOT FPRIME_DISABLE_DEFAULT_LOGGER)
@@ -42,7 +53,6 @@ if (FPRIME_USE_POSIX)
         "${CMAKE_CURRENT_LIST_DIR}/Pthreads/BufferQueueCommon.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Pthreads/PriorityBufferQueue.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Pthreads/MaxHeap/MaxHeap.cpp"
-        "${CMAKE_CURRENT_LIST_DIR}/Linux/File.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Posix/Task.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Linux/InterruptLock.cpp"
         "${CMAKE_CURRENT_LIST_DIR}/Linux/WatchdogTimer.cpp"
@@ -90,20 +100,16 @@ if (FPRIME_USE_BAREMETAL_SCHEDULER)
     list(APPEND SOURCE_FILES "${CMAKE_CURRENT_LIST_DIR}/Baremetal/Task.cpp")
 endif()
 register_fprime_module()
-
-# Add stubs directory for testing builds
-if (BUILD_TESTING)
-    add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/Stubs/")
-endif()
+require_fprime_implementation(Os/File)
 
 ### UTS ### Note: 3 separate UTs registered here.
 set(UT_SOURCE_FILES
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsQueueTest.cpp"
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsTestMain.cpp"
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/IntervalTimerTest.cpp"
+  "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsFileSystemTest.cpp"
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsValidateFileTest.cpp"
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsTaskTest.cpp"
-  "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsFileSystemTest.cpp"
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsSystemResourcesTest.cpp"
   "${CMAKE_CURRENT_LIST_DIR}/test/ut/OsMutexBasicLockableTest.cpp"
 )