diff --git a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp index 423f2a234..e472357d8 100644 --- a/cpp/csp/adapters/websocket/ClientAdapterManager.cpp +++ b/cpp/csp/adapters/websocket/ClientAdapterManager.cpp @@ -52,8 +52,15 @@ void ClientAdapterManager::start( DateTime starttime, DateTime endtime ) if( m_inputAdapter ) { m_endpoint -> setOnMessage( [ this ]( void* c, size_t t ) { - PushBatch batch( m_engine -> rootEngine() ); - m_inputAdapter -> processMessage( c, t, &batch ); + try + { + PushBatch batch( m_engine -> rootEngine() ); + m_inputAdapter -> processMessage( c, t, &batch ); + } + catch( csp::Exception & err ) + { + pushStatus( StatusLevel::ERROR, ClientStatusType::GENERIC_ERROR, err.what() ); + } } ); } else { diff --git a/cpp/csp/cppnodes/baselibimpl.cpp b/cpp/csp/cppnodes/baselibimpl.cpp index 52a5537d9..c36393b98 100644 --- a/cpp/csp/cppnodes/baselibimpl.cpp +++ b/cpp/csp/cppnodes/baselibimpl.cpp @@ -705,6 +705,9 @@ DECLARE_CPPNODE( struct_fromts ) ); } + if( unlikely( !out.get() -> validate() ) ) + CSP_THROW( ValueError, "Struct " << cls.value() -> name() << " is not valid; required fields " << out -> formatAllUnsetStrictFields() << " did not tick" ); + CSP_OUTPUT( std::move( out ) ); } @@ -758,6 +761,9 @@ DECLARE_CPPNODE( struct_collectts ) } ); } + + if( unlikely( !out.get() -> validate( ) ) ) + CSP_THROW( ValueError, "Struct " << cls.value() -> name() << " is not valid; some required fields did not tick" ); CSP_OUTPUT( std::move( out ) ); } diff --git a/cpp/csp/engine/InputAdapter.h b/cpp/csp/engine/InputAdapter.h index f992d6ff2..051ca83ea 100644 --- a/cpp/csp/engine/InputAdapter.h +++ b/cpp/csp/engine/InputAdapter.h @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace csp @@ -55,6 +56,12 @@ class InputAdapter : public TimeSeriesProvider, public EngineOwned template bool InputAdapter::consumeTick( const T & value ) { + if constexpr( CspType::Type::fromCType::type == CspType::TypeTraits::STRUCT ) + { + if( unlikely( !( value -> validate() ) ) ) + CSP_THROW( ValueError, "Struct " << value -> meta() -> name() << " is not valid; required fields " << value -> formatAllUnsetStrictFields() << " were not set on init" ); + } + switch( pushMode() ) { case PushMode::LAST_VALUE: diff --git a/cpp/csp/engine/Struct.cpp b/cpp/csp/engine/Struct.cpp index 42830357e..3c327aee7 100644 --- a/cpp/csp/engine/Struct.cpp +++ b/cpp/csp/engine/Struct.cpp @@ -1,20 +1,27 @@ #include #include #include +#include +#include +#include namespace csp { StructField::StructField( CspTypePtr type, const std::string & fieldname, - size_t size, size_t alignment ) : + size_t size, size_t alignment, bool isOptional ) : m_fieldname( fieldname ), m_offset( 0 ), m_size( size ), m_alignment( alignment ), m_maskOffset( 0 ), + m_noneMaskOffset( 0 ), m_maskBit( 0 ), + m_noneMaskBit( 0 ), m_maskBitMask( 0 ), - m_type( type ) + m_noneMaskBitMask( 0 ), + m_type( type ), + m_isOptional( isOptional ) { } @@ -33,11 +40,12 @@ and adjustments required for the hidden fields */ -StructMeta::StructMeta( const std::string & name, const Fields & fields, - std::shared_ptr base ) : m_name( name ), m_base( base ), m_fields( fields ), +StructMeta::StructMeta( const std::string & name, const Fields & fields, bool isStrict, + std::shared_ptr base ) : m_name( name ), m_base( base ), m_fields( fields ), m_size( 0 ), m_partialSize( 0 ), m_partialStart( 0 ), m_nativeStart( 0 ), m_basePadding( 0 ), m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), m_firstNativePartialField( 0 ), - m_isPartialNative( true ), m_isFullyNative( true ) + m_optionalFieldsSetBits( nullptr ), m_optionalFieldsNoneBits( nullptr ), m_isPartialNative( true ), + m_isFullyNative( true ), m_isStrict( isStrict ) { if( m_fields.empty() && !m_base) CSP_THROW( TypeError, "Struct types must define at least 1 field" ); @@ -95,20 +103,70 @@ StructMeta::StructMeta( const std::string & name, const Fields & fields, //Setup masking bits for our fields //NOTE we can be more efficient by sticking masks into any potential alignment gaps, dont want to spend time on it //at this point - m_maskSize = !m_fields.empty() ? 1 + ( ( m_fields.size() - 1 ) / 8 ) : 0; + + size_t optionalFieldCount = std::count_if( m_fields.begin(), m_fields.end(), []( const auto & f ) { return f -> isOptional(); } ); + + m_maskSize = !m_fields.empty() ? 1 + ( ( m_fields.size() + optionalFieldCount - 1 ) / 8 ) : 0; m_size = offset + m_maskSize; m_partialSize = m_size - baseSize; m_maskLoc = m_size - m_maskSize; + + uint8_t numRemainingBits = ( m_fields.size() - m_firstPartialField + optionalFieldCount ) % 8; + m_lastByteMask = ( 1u << numRemainingBits ) - 1; size_t maskLoc = m_maskLoc; uint8_t maskBit = 0; - for( auto & f : m_fields ) + + // Set optional fields first so that their 2-bits never cross a byte boundary + m_optionalFieldsSetBits = new uint8_t[ m_maskSize ](); + m_optionalFieldsNoneBits = new uint8_t[ m_maskSize ](); + for( size_t i = 0; i < m_fields.size(); ++i ) + { + auto & f = m_fields[ i ]; + if( f -> isOptional() ) + { + f -> setMaskOffset( maskLoc, maskBit ); + f -> setNoneMaskOffset( maskLoc, ++maskBit ); // use adjacent bit for None bit + if( ++maskBit == 8 ) + { + m_optionalFieldsSetBits[ maskLoc - m_maskLoc ] = 0xAA; + m_optionalFieldsNoneBits[ maskLoc - m_maskLoc ] = 0x55; + maskBit = 0; + ++maskLoc; + } + } + } + // deal with last (partial) byte filled with optional fields + auto lastOptIndex = maskLoc - m_maskLoc; + switch( maskBit / 2 ) { - f -> setMaskOffset( maskLoc, maskBit ); - if( ++maskBit == 8 ) + case 1: + m_optionalFieldsSetBits[ lastOptIndex ] = 0x1; + m_optionalFieldsNoneBits[ lastOptIndex ] = 0x2; + break; + case 2: + m_optionalFieldsSetBits[ lastOptIndex ] = 0x5; + m_optionalFieldsNoneBits[ lastOptIndex ] = 0xA; + break; + case 3: + m_optionalFieldsSetBits[ lastOptIndex ] = 0x15; + m_optionalFieldsNoneBits[ lastOptIndex ] = 0x2A; + break; + default: + break; // default initialized to 0 + } + + for( size_t i = 0; i < m_fields.size(); ++i ) + { + auto & f = m_fields[ i ]; + if( !f -> isOptional() ) { - maskBit = 0; - ++maskLoc; + f -> setMaskOffset( maskLoc, maskBit ); + if( ++maskBit == 8 ) + { + maskBit = 0; + ++maskLoc; + } } } @@ -128,11 +186,22 @@ StructMeta::StructMeta( const std::string & name, const Fields & fields, if( !rv.second ) CSP_THROW( ValueError, "csp Struct " << name << " attempted to add existing field " << m_fields[ idx ] -> fieldname() ); } + + // The complete inheritance hierarchy must agree on strict/non-strict + if( m_base && m_isStrict != m_base -> isStrict() ) + { + CSP_THROW( ValueError, + "Struct " << m_name << " was declared " << ( m_isStrict ? "strict" : "non-strict" ) << " but derives from " + << m_base -> name() << " which is " << ( m_base -> isStrict() ? "strict" : "non-strict" ) + ); + } } StructMeta::~StructMeta() { m_default.reset(); + delete[] m_optionalFieldsSetBits; + delete[] m_optionalFieldsNoneBits; } Struct * StructMeta::createRaw() const @@ -453,23 +522,62 @@ void StructMeta::clear( Struct * s ) const m_base -> clear( s ); } +std::string StructMeta::formatAllUnsetStrictFields( const Struct * s ) const +{ + bool first = true; + std::stringstream ss; + ss << "["; + + for( size_t i = 0; i < m_fields.size(); ++i ) + { + const auto & f = m_fields[ i ]; + if( !f -> isSet( s ) && !f -> isNone( s ) ) + { + if( !first ) + ss << ", "; + else + first = false; + ss << f -> fieldname(); + } + } + ss << "]"; + return ss.str(); +} + bool StructMeta::allFieldsSet( const Struct * s ) const { - size_t numLocalFields = m_fields.size() - m_firstPartialField; - uint8_t numRemainingBits = numLocalFields % 8; + /* + We can use bit operations to validate the struct. + 1. Let M1 be the bitmask with 1s that align with the set bit of optional fields and + 2. Let M2 be the bitmask with 1s that align with the none bit of optional fields. + -- Both M1 and M2 are computed trivially when we create the meta. + 3. Let M1* = M1 & mask. M1* now is the bitmask of all set optional fields. + 4. Similarly, let M2* = M2 & mask, such that M2* is the bitmask of all none optional fields. + 5. Let M3 = mask | (M1* << 1) | (M2* >> 1). Since the shifted set/none bitsets will fill in that optional fields other bit, + the struct can validate iff M3 is all 1s. + + Notes: + 1) We do this on a byte by byte basis currently. If we add 32/64 bit padding to the struct mask, we could do this as one single step for most structs. + 2) There is an edge condition if a) the set bit of an optional field is the last bit of a byte or b) the none bit of an optional field is the first bit of a byte. + So, when we create the meta, we ensure this never happens by being smart about the ordering of fields in the mask. + */ const uint8_t * m = reinterpret_cast( s ) + m_maskLoc; - const uint8_t * e = m + m_maskSize - bool( numRemainingBits ); - for( ; m < e; ++m ) + const uint8_t * e = m + m_maskSize - bool( m_lastByteMask ); + + const uint8_t * M1 = m_optionalFieldsSetBits; + const uint8_t * M2 = m_optionalFieldsNoneBits; + + for( ; m < e; ++m, ++M1, ++M2 ) { - if( *m != 0xFF ) + if( ( *m | ( ( *m & *M1 ) << 1 ) | ( ( *m & *M2 ) >> 1 ) ) != 0xFF ) return false; } - if( numRemainingBits > 0 ) + if( m_lastByteMask ) { - uint8_t bitmask = ( 1u << numRemainingBits ) - 1; - if( ( *m & bitmask ) != bitmask ) + uint8_t masked = *m & m_lastByteMask; + if( ( masked | ( ( ( masked & *M1 ) << 1 ) & m_lastByteMask ) | ( ( masked & *M2 ) >> 1 ) ) != m_lastByteMask ) return false; } @@ -494,6 +602,20 @@ void StructMeta::destroy( Struct * s ) const m_base -> destroy( s ); } +[[nodiscard]] bool StructMeta::validate( const Struct * s ) const +{ + if( !s -> meta() -> isStrict() ) + return true; + + for ( const StructMeta * cur = this; cur; cur = cur -> m_base.get() ) + { + if ( !cur -> allFieldsSet( s ) ) + return false; + } + + return true; +} + Struct::Struct( const std::shared_ptr & meta ) { //Initialize meta shared_ptr diff --git a/cpp/csp/engine/Struct.h b/cpp/csp/engine/Struct.h index 64b51ecae..3bd312284 100644 --- a/cpp/csp/engine/Struct.h +++ b/cpp/csp/engine/Struct.h @@ -22,6 +22,8 @@ class StructField { public: + static constexpr uint8_t BITS_PER_BYTE = 8; + virtual ~StructField() {} const std::string & fieldname() const { return m_fieldname; } @@ -30,27 +32,55 @@ class StructField size_t size() const { return m_size; } //size of field in bytes size_t alignment() const { return m_alignment; } //alignment of the field size_t maskOffset() const { return m_maskOffset; } //offset to location of the mask byte fo this field from start of struct mem - uint8_t maskBit() const { return m_maskBit; } //bit within mask byte associated with this field + uint8_t maskBit() const { return m_maskBit; } //bit within mask byte associated with this field. Just used for debugging uint8_t maskBitMask() const { return m_maskBitMask; } //same as maskBit but as a mask ( 1 << bit + size_t noneMaskOffset() const { return m_noneMaskOffset; } //same fields as above but for the None bit, if this an optional field + uint8_t noneMaskBit() const { return m_noneMaskBit; } + uint8_t noneMaskBitMask() const { return m_noneMaskBitMask; } bool isNative() const { return m_type -> type() <= CspType::Type::MAX_NATIVE_TYPE; } + bool isOptional() const { return m_isOptional; } void setOffset( size_t off ) { m_offset = off; } void setMaskOffset( size_t off, uint8_t bit ) { - CSP_ASSERT( bit < 8 ); + CSP_ASSERT( bit < BITS_PER_BYTE ); m_maskOffset = off; m_maskBit = bit; m_maskBitMask = 1 << bit; } + void setNoneMaskOffset( size_t off, uint8_t bit ) + { + CSP_ASSERT( m_isOptional ); + CSP_ASSERT( bit < BITS_PER_BYTE ); + + m_noneMaskOffset = off; + m_noneMaskBit = bit; + m_noneMaskBitMask = 1 << bit; + } + bool isSet( const Struct * s ) const { const uint8_t * m = reinterpret_cast( s ) + m_maskOffset; return (*m ) & m_maskBitMask; } + bool isNone( const Struct * s ) const + { + const uint8_t * m = reinterpret_cast( s ) + m_noneMaskOffset; + return ( *m ) & m_noneMaskBitMask; + } + + void setNone( Struct * s ) const + { + CSP_ASSERT( m_isOptional ); + + uint8_t * m = reinterpret_cast( s ) + m_noneMaskOffset; + (*m) |= m_noneMaskBitMask; + } + //copy methods need not deal with mask set/unset, only copy values virtual void copyFrom( const Struct * src, Struct * dest ) const = 0; @@ -75,12 +105,14 @@ class StructField protected: StructField( CspTypePtr type, const std::string & fieldname, - size_t size, size_t alignment ); + size_t size, size_t alignment, bool optional ); void setIsSet( Struct * s ) const { uint8_t * m = reinterpret_cast( s ) + m_maskOffset; (*m) |= m_maskBitMask; + + clearIsNone( s ); // no-op if not an optional field } const void * valuePtr( const Struct * s ) const @@ -99,15 +131,25 @@ class StructField (*m) &= ~m_maskBitMask; } + void clearIsNone( Struct * s ) const + { + uint8_t * m = reinterpret_cast( s ) + m_noneMaskOffset; + (*m) &= ~m_noneMaskBitMask; + } + private: std::string m_fieldname; size_t m_offset; const size_t m_size; const size_t m_alignment; size_t m_maskOffset; + size_t m_noneMaskOffset; uint8_t m_maskBit; + uint8_t m_noneMaskBit; uint8_t m_maskBitMask; + uint8_t m_noneMaskBitMask; CspTypePtr m_type; + bool m_isOptional; }; using StructFieldPtr = std::shared_ptr; @@ -120,7 +162,7 @@ class NativeStructField : public StructField public: NativeStructField() {} - NativeStructField( const std::string & fieldname ) : NativeStructField( CspType::fromCType::type(), fieldname ) + NativeStructField( const std::string & fieldname, bool optional ) : NativeStructField( CspType::fromCType::type(), fieldname, optional ) { } @@ -157,7 +199,8 @@ class NativeStructField : public StructField } protected: - NativeStructField( CspTypePtr type, const std::string & fieldname ) : StructField( type, fieldname, sizeof( T ), alignof( T ) ) + NativeStructField( CspTypePtr type, const std::string & fieldname, bool optional ) + : StructField( type, fieldname, sizeof( T ), alignof( T ), optional ) {} }; @@ -179,7 +222,7 @@ using TimeStructField = NativeStructField