PackedMemoryBuilder.hxx

Go to the documentation of this file.

// Copyright 2022 Siemens Digital Industries Software
// ==================================================
// Copyright 2016.
// Siemens Product Lifecycle Management Software Inc.
// All Rights Reserved.
// ==================================================
// Copyright 2022 Siemens Digital Industries Software

#ifndef PACKED_MEMORY_BUILDER_HXX
#define PACKED_MEMORY_BUILDER_HXX

#include <map>
#include <set>
#include <string.h>
#include <vector>
#include <base_utils/base_utils_errors.h>
#include <base_utils/IFail.hxx>
#include <base_utils/Mem.h>
#include <base_utils/SharedPtr.hxx>
        
namespace Teamcenter
{
    namespace PackedMemory
    {

inline size_t getAlignedSize(
    size_t datasize     
    )
{
    // This should give us the machine alignment on a pointer.
    const struct { char c; char *ptr; } *pAlignment= 0;
    const size_t alignment= size_t(&pAlignment->ptr);

    return (datasize + alignment - 1) / alignment * alignment;
}

struct DefaultAllocator
{
    void * alloc(
        size_t size 
        ) const
    {
        // Forward to MEM_alloc to do the allocation.
        return MEM_alloc((int)size);
    }

    void *alloc_packed(
        size_t dataToPackCount,
        MEM_data_to_pack_p_t pDataToPack,
        logical allowOverride
        ) const
    {
        // Forward to MEM_alloc_packed to do the allocation.
        return MEM_alloc_packed(dataToPackCount, pDataToPack, allowOverride);
    }
    
    void free(
        void *pAddress 
        ) const
    {
        // Forward to MEM_alloc to do the allocation.
        MEM_free(pAddress);
    }
};

enum CreateMode
{
    AllowOverride,      
    NoOverride,         
    NoMarking           
};



template <class T, typename Alloc = DefaultAllocator> 
class Builder
{
public:
    explicit Builder(size_t elements= 1);
    
    explicit Builder(const char *pSrcString, size_t maxLength= ~0);

    T *get() { return m_pObjectData; }

    const T *get() const { return m_pObjectData; }

    void setLookupStringData(bool lookupStringData) { m_lookupStringData= lookupStringData; }

    bool getLookupStringData() const { return m_lookupStringData; }

    template <typename K> 
    inline K addString(K &pDestPtr, const char *pSrcString);

    template <typename K> 
    K addData(K &pDestPtr, size_t elements= 1, const void *pSourceData= NULL);

    template <typename K, typename J>
    K addReference(K &pDestPtr, J &pReferenceData);

    void createBuffer(T **pDestination, CreateMode createMode = NoMarking) const;

    Alloc& allocator() { return m_allocator; }

    const Alloc& allocator() const { return m_allocator; }
    
private:
    Builder(const Builder &);
    Builder & operator = (const Builder &);

    struct BackRefData
    {
        BackRefData() : pData(NULL), datasize(0) {}
        BackRefData(void *_pData, size_t _datasize) : pData(_pData), datasize(_datasize) {}

        void  *pData;               
        size_t datasize;            
    };

    struct StringData
    {
        StringData() : m_pString(NULL), m_datasize(0) {}
        StringData(const char *pString, size_t datasize) : m_pString(pString), m_datasize(datasize) {}

        bool operator < (const StringData &rhs) const 
        { 
            return m_datasize < rhs.m_datasize || 
                (m_datasize == rhs.m_datasize && m_pString && rhs.m_pString && memcmp(m_pString, rhs.m_pString, m_datasize) < 0);
        }

        const char *m_pString;            
        size_t      m_datasize;           
    };



    typedef std::vector<char> Buffer;                                       
    typedef std::vector<void *> ReferencedObjectsList;                      
    typedef std::map<void *, Buffer> ObjectDataMap;                         
    typedef std::map<void *, ReferencedObjectsList> ReferencedObjectsMap;   
    typedef std::map<void *, void *> ObjectPointerMap;                      
    typedef std::map<const void *, BackRefData> ObjectBackRefDataMap;       
    typedef std::set<StringData> StringDataSet;                             



    ObjectDataMap m_objectDataMap;                  
    ObjectBackRefDataMap m_objectBackRefDataMap;    
    ReferencedObjectsMap m_referencedObjects;       
    ObjectPointerMap m_backReferencedObjects;       
    StringDataSet m_addedStringData;                

    T *m_pObjectData;                               
    Alloc m_allocator;                              
    bool m_lookupStringData;                        

    bool getMappedObject(const void *pBaseAddress, void *&pMappedObject, bool bBaseOnly= false) const;

    void createOutputBuffer(ObjectPointerMap &destinationMap, T **pDestination) const;

    void createOutputBufferUsingSMPack(ObjectPointerMap &destinationMap, T **pDestination, logical allowOverride) const;
};

template <class T, typename Alloc>
Builder<T, Alloc>::Builder(size_t elements):
    m_pObjectData(NULL), m_lookupStringData(false)
{ 
    // Figure out the size we need.
    size_t datasize= getAlignedSize(sizeof(T) * elements);

    // Create the buffer for the object(s) T we are storing.
    // We use NULL to indicate the base object. This should now always
    // be ordered first in the map wich is important.
    Buffer &buffer= m_objectDataMap[NULL];
    buffer.resize(datasize);

    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]]= BackRefData(NULL, datasize);

    // Assign the data to the buffer for quick access.
    m_pObjectData= datasize ? reinterpret_cast<T *>(&buffer[0]) : NULL;
}

template <class T, typename Alloc>
Builder<T, Alloc>::Builder(const char *pSrcString, size_t maxLength):
    m_pObjectData(NULL), m_lookupStringData(false)
{ 
    // What the size we need for this string.
    const size_t stringSize= pSrcString ? strlen(pSrcString) + 1 : 0;
    const size_t datasize= stringSize < maxLength ? stringSize : maxLength;
    
    // Create the buffer for the object(s) T we are storing.
    // We use NULL to indicate the base object. This should now always
    // be ordered first in the map wich is important.
    Buffer &buffer= m_objectDataMap[NULL];
    buffer.resize(datasize);

    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]]= BackRefData(NULL, datasize);
    
    // Assign the data to the buffer for quick access.
    m_pObjectData= datasize ? reinterpret_cast<T *>(&buffer[0]) : NULL;

    // If we have the data to copy, do it now.
    if (pSrcString && datasize)
    {
        strncpy(&buffer[0], pSrcString, datasize);
        buffer[datasize - 1] = '\0';
    }
}

template <class T, typename Alloc> template <typename K> 
K Builder<T, Alloc>::addString(K &pDestPtr, const char *pSrcString)
{
    // What the size we need for this string.
    const size_t datasize= pSrcString ? strlen(pSrcString) + 1 : 0;

    // If we have a string,
    if (datasize && m_lookupStringData)
    {
        // see if we can find the string in our string reference set.
        typename StringDataSet::iterator searchIter = m_addedStringData.find(StringData(pSrcString, datasize));
        // If we already have seen this string,
        if (searchIter != m_addedStringData.end())
        {
            // start by casting the found string to the proper output type.
            K pReferenceData = reinterpret_cast<K>(const_cast<char *>(searchIter->m_pString));
            // Now we can add the reference to the found string to the destination and return value.
            return addReference(pDestPtr, pReferenceData);
        }
        else
        {
            // We have not seen this before so forward having the pSourceData as the string data.
            K pAddedData = addData(pDestPtr, datasize, pSrcString);
            // Make sure we add the string into the StringReferenceData set (we do handle nulls).
            m_addedStringData.insert(StringData(pAddedData, datasize));

            // Return the string we added.
            return pAddedData;
        }
    }

    // This is either a NULL pointer or we are not doing lookups in the string collection set.
    // Just forward to addData having the pSourceData as the string data.
    return addData(pDestPtr, datasize, pSrcString);
}

template <class T, typename Alloc> template <typename K>
K Builder<T, Alloc>::addData(K &pDestPtr, size_t elements, const void *pSourceData)
{
    // What the size we need for these referenced data.
    const size_t datasize= getAlignedSize(sizeof(*pDestPtr) * elements);

    // The address of the parameter K &pDestPtr.
    void *pParameterAddess= (void *)&pDestPtr;

    // Make sure that we have a pointer address that we have seen before. We can't add data
    // unless we reference some of the external data buffers.
    void *pBaseObject= NULL;
    if (!getMappedObject(pParameterAddess, pBaseObject))
    { 
        throw IFail( BASE_UTILS_invalid_reference ); 
    }

    // See if we have seen this before, this is most likely a coding error.
    ObjectDataMap::const_iterator searchIter= m_objectDataMap.find(pParameterAddess);
    if (searchIter != m_objectDataMap.end())
    {
        throw IFail( BASE_UTILS_already_initialized, "m_objectDataMap" ); 
    }

    // Add new allocation we did for this base object.
    m_referencedObjects[pBaseObject].push_back(pParameterAddess);

    // Create the buffer for the object(s) T we are storing.
    Buffer &buffer= m_objectDataMap[pParameterAddess];
    buffer.resize(datasize);
    
    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]]= BackRefData(pParameterAddess, datasize);

    // If we have source data,
    if (pSourceData && datasize)
    {
        // copy the data to the destination.
        memcpy(&buffer[0], pSourceData, datasize);
    }

    // Set the pointer to the buffer we have allocated.
    pDestPtr= datasize ? reinterpret_cast<K>(&buffer[0]) : NULL;

    // Give out the pointer to the caller.
    return pDestPtr;
}

template <class T, typename Alloc> template <typename K, typename J>
K Builder<T, Alloc>::addReference(K &pDestPtr, J &pReferenceData)
{
    // The address of the parameter K &pDestPtr.
    void *pParameterAddess= (void *)&pDestPtr;

    // Make sure that we have a pointer address that we have seen before. We can't add data
    // unless we reference some of the external data buffers.
    void *pBaseObject= NULL;
    if (!getMappedObject(pParameterAddess, pBaseObject))
    { 
        throw IFail( BASE_UTILS_invalid_reference ); 
    }

    // See if we have seen this before, this is most likely a coding error.
    ObjectDataMap::const_iterator searchIter= m_objectDataMap.find(pParameterAddess);
    if (searchIter != m_objectDataMap.end())
    {
        throw IFail( BASE_UTILS_already_initialized, "m_objectDataMap" ); 
    }

    // Make sure that we have a pointer address that we have seen before. We can't add data
    // unless we reference some of the external data buffers that we have created.
    void *pRefBaseObject= NULL;
    if (!getMappedObject(pReferenceData, pRefBaseObject, true))
    { 
        throw IFail( BASE_UTILS_invalid_reference ); 
    }

    // Add new allocation we did for this base object.
    m_referencedObjects[pBaseObject].push_back(pParameterAddess);

    // Add new back reference to the existing buffer.
    m_backReferencedObjects[pParameterAddess]= pRefBaseObject;
    
    // Create the buffer for the object(s) T we are storing with zero length.
    // This will indicate that we should look in the back reference for the data.
    Buffer &buffer= m_objectDataMap[pParameterAddess];
    buffer.resize(0);

    // Also populate the back reference.
    m_objectBackRefDataMap[&buffer[0]] = BackRefData(pParameterAddess, 0);

    // Set the desination pointer to the reference data.
    pDestPtr= pReferenceData;

    // Give out the pointer to the caller.
    return pDestPtr;
}

template <class T, typename Alloc>
void Builder<T, Alloc>::createBuffer(T **pDestination, CreateMode createMode) const
{
    // Create the output data and give the object mapping back to us.
    ObjectPointerMap destinationMap;
    
    // See how we are creating the buffer.
    if (createMode == NoMarking)
    {
        // This will create a tight packing.
        createOutputBuffer(destinationMap, pDestination);
    }
    else
    {
        // Create a packed buffer with marked allocation (unless override is enabled). 
        createOutputBufferUsingSMPack(destinationMap, pDestination, createMode == AllowOverride);
    }

    // Loop over all the extra reference object we have added.
    ReferencedObjectsMap::const_iterator referencedIter= m_referencedObjects.begin();
    for ( ; referencedIter != m_referencedObjects.end(); ++referencedIter)
    {
        // Find the mapped Object Data for this reference.
        ObjectDataMap::const_iterator objectDataIter= m_objectDataMap.find(referencedIter->first);
        if (objectDataIter == m_objectDataMap.end())
        {
            throw IFail( BASE_UTILS_expected_data_missing, "m_objectDataMap" ); 
        }

        // We also need to find this object in the output buffer.
        ObjectPointerMap::iterator destinationObject= destinationMap.find(referencedIter->first);
        if (destinationObject == destinationMap.end())
        {
            throw IFail( BASE_UTILS_expected_data_missing, "destinationMap" ); 
        }

        // Loop over all the objects we reference to this object and update the desination.
        ReferencedObjectsList::const_iterator childIter= referencedIter->second.begin();
        for ( ; childIter != referencedIter->second.end(); ++childIter)
        {
            // Find this object in the destination map.
            ObjectPointerMap::iterator referencedDestinationObject= destinationMap.find(*childIter);
            if (referencedDestinationObject == destinationMap.end())
            {
                throw IFail( BASE_UTILS_expected_data_missing, "destinationMap" ); 
            }

            // Calculate the offset where this object is located in the referenced data structure.
            size_t offset= size_t(*childIter) - size_t(&objectDataIter->second[0]);

            // Now we can re-point that pointer to the new destination location.
            *reinterpret_cast<void **>(size_t(destinationObject->second) + offset) = referencedDestinationObject->second;
        }
    }
}


template <class T, typename Alloc>
bool Builder<T, Alloc>::getMappedObject(const void *pBaseAddress, void *&pMappedObject, bool bBaseOnly) const
{
    // If we have a base address and we have something mapped,
    if (pBaseAddress != NULL && !m_objectBackRefDataMap.empty())
    {
        // First find the address. Try to find an address that is the next bigger pointer.
        typename ObjectBackRefDataMap::const_iterator searchIter = m_objectBackRefDataMap.upper_bound(pBaseAddress);
        // If we are not at the beginning, move to the prior element. This should be the object for the pBaseAddress.
        if (searchIter != m_objectBackRefDataMap.begin()) { --searchIter; }

        // Figure out the end pointer address for the buffer.
        const void *pObjectEnd = reinterpret_cast<const char *>(searchIter->first) + searchIter->second.datasize;

        // If the address is within the object, the reference is good.
        if ((bBaseOnly && pBaseAddress == searchIter->first) || (searchIter->first <= pBaseAddress && pBaseAddress < pObjectEnd))
        {
            // Assign the pMappedObject with the pData value we mapped and return true.
            pMappedObject = searchIter->second.pData;
            return true;
        }
    }

    // Could not find a object that encapsulate the address. Assign NULL to pMappedObject and return false.
    pMappedObject = NULL;
    return false;
}

template <class T, typename Alloc>
void Builder<T, Alloc>::createOutputBuffer(ObjectPointerMap &destinationMap, T **pDestination) const
{
    // Make sure we have an output pointer.
    if (!pDestination)
    { 
        throw IFail( BASE_UTILS_invalid_parameter ); 
    }

    // Size of the buffer we need.
    size_t totalSize= 0;

    // Count up all of the buffers we have allocated including the our object...
    ObjectDataMap::const_iterator objectDataIter= m_objectDataMap.begin();
    for ( ; objectDataIter != m_objectDataMap.end(); ++objectDataIter)
    {
        totalSize+= objectDataIter->second.size();
    }

    // Allocate the output buffer using the specific Alloc. 
    char *pOutputBuffer= totalSize ? reinterpret_cast<char *>(allocator().alloc(totalSize)) : NULL;
    // Make sure we have a buffer if expected.
    if (totalSize && !pOutputBuffer)
    {
        throw IFail( BASE_UTILS_no_memory ); 
    }

    // We assing this now to the destination so we have a better chance not to leak the memory.
    // Just incase we throw an exception while processing the data to the buffer.    
    *pDestination= totalSize ? reinterpret_cast<T *>(pOutputBuffer) : NULL;

    // This is the end of the buffer where the "pOutputBuffer" should be 
    // after we have processed all the data, if not, we failed.
    char *pOutputBufferEnd= pOutputBuffer + totalSize;

    // Now we can start fixing up all the pointer data.
    objectDataIter= m_objectDataMap.begin();
    for ( ; objectDataIter != m_objectDataMap.end(); ++objectDataIter)
    {
        // If we don't have any data,
        if (objectDataIter->second.empty())
        {
            // just set the destination as NULL.
            destinationMap[objectDataIter->first]= NULL;
        }
        else
        {
            // otherwise copy the data and move the output buffer pointer.
            memcpy(pOutputBuffer, &objectDataIter->second[0], objectDataIter->second.size());
            destinationMap[objectDataIter->first]= pOutputBuffer;
            pOutputBuffer+= objectDataIter->second.size();
        }
    }


    // Also add all the back reference data.
    ObjectPointerMap::const_iterator backRefIter= m_backReferencedObjects.begin();
    for ( ; backRefIter != m_backReferencedObjects.end(); ++backRefIter)
    {
        // Find the back reference data in the destination. This should be NULL.
        ObjectPointerMap::iterator backRefDataIter= destinationMap.find(backRefIter->first);
        if (backRefDataIter == destinationMap.end() || backRefDataIter->second != NULL)
        {
            throw IFail( BASE_UTILS_internal_error );
        }

        // We should also be able to find referenced data in the destination.
        ObjectPointerMap::const_iterator searchIter= destinationMap.find(backRefIter->second);
        if (searchIter == destinationMap.end() || searchIter->second == NULL)
        {
            throw IFail( BASE_UTILS_internal_error );
        }

        // Add another destination mapping to exiting data.
        backRefDataIter->second= searchIter->second;
    }

    // Cross the "T's" :)... We should have moved the pointer as expected and the "NULL"
    // mapped object pointing to the beginning of the buffer.
    if (pOutputBuffer != pOutputBufferEnd || destinationMap[NULL] != *pDestination)
    {
        throw IFail( BASE_UTILS_internal_error );
    }
}

template <class T, typename Alloc>
void Builder<T, Alloc>::createOutputBufferUsingSMPack(ObjectPointerMap &destinationMap, T **pDestination, logical allowOverride) const
{
    // Make sure we have an output pointer.
    if (!pDestination)
    {
        throw IFail(BASE_UTILS_invalid_parameter);
    }

    // This will hold the information we are packing.
    std::vector<MEM_data_to_pack_t> dataToPack;
    dataToPack.reserve(m_objectDataMap.size());

    // Count up all of the buffers we have allocated including the our object...
    ObjectDataMap::const_iterator objectDataIter = m_objectDataMap.begin();
    for (; objectDataIter != m_objectDataMap.end(); ++objectDataIter)
    {
        MEM_data_to_pack_t data;
        data.data = !objectDataIter->second.empty() ? objectDataIter->second.data() : NULL;
        data.data_size = objectDataIter->second.size();

        dataToPack.push_back(data);
    }

    // We assing the allocation to the destination so we have a better chance not to leak the memory.
    // Just incase we throw an exception while processing the data to the buffer.    
    *pDestination = reinterpret_cast<T *>(allocator().alloc_packed(dataToPack.size(), &dataToPack[0], allowOverride));
    // Make sure we have a buffer if expected.
    if (dataToPack.front().data_size && !*pDestination)
    {
        throw IFail(BASE_UTILS_no_memory);
    }

    // Now we can start fixing up all the pointer data.
    objectDataIter = m_objectDataMap.begin();
    for (size_t i = 0; objectDataIter != m_objectDataMap.end(); ++i, ++objectDataIter)
    {
        destinationMap[objectDataIter->first] = (void *)dataToPack[i].data;
    }

    // Also add all the back reference data.
    ObjectPointerMap::const_iterator backRefIter = m_backReferencedObjects.begin();
    for (; backRefIter != m_backReferencedObjects.end(); ++backRefIter)
    {
        // Find the back reference data in the destination. This should be NULL.
        ObjectPointerMap::iterator backRefDataIter = destinationMap.find(backRefIter->first);
        if (backRefDataIter == destinationMap.end() || backRefDataIter->second != NULL)
        {
            throw IFail(BASE_UTILS_internal_error);
        }

        // We should also be able to find referenced data in the destination.
        ObjectPointerMap::const_iterator searchIter = destinationMap.find(backRefIter->second);
        if (searchIter == destinationMap.end() || searchIter->second == NULL)
        {
            throw IFail(BASE_UTILS_internal_error);
        }

        // Add another destination mapping to exiting data.
        backRefDataIter->second = searchIter->second;
    }
}

} // namespace PackedMemory
} // namespace Teamcenter

#endif // PACKED_MEMORY_BUILDER_HXX