Updated VertexBuffer's getSpaceRequired and storeVertexAttributes methods to return bools and fixed some validation of parameters to prevent under and overflows.
TRAC #23643
Signed-off-by: Nicolas Capens
Signed-off-by: Shannon Woods
Author: Geoff Lang
diff --git a/src/libGLESv2/renderer/IndexBuffer.cpp b/src/libGLESv2/renderer/IndexBuffer.cpp
index 2ac9446..37dbd3e 100644
--- a/src/libGLESv2/renderer/IndexBuffer.cpp
+++ b/src/libGLESv2/renderer/IndexBuffer.cpp
@@ -67,18 +67,30 @@
return mIndexBuffer->getSerial();
}
-int IndexBufferInterface::mapBuffer(unsigned int size, void** outMappedMemory)
+bool IndexBufferInterface::mapBuffer(unsigned int size, void** outMappedMemory, unsigned int *streamOffset)
{
- if (!mIndexBuffer->mapBuffer(mWritePosition, size, outMappedMemory))
+ // Protect against integer overflow
+ if (mWritePosition + size < mWritePosition)
{
- *outMappedMemory = NULL;
- return -1;
+ return false;
}
- int oldWritePos = static_cast<int>(mWritePosition);
- mWritePosition += size;
+ if (!mIndexBuffer->mapBuffer(mWritePosition, size, outMappedMemory))
+ {
+ if (outMappedMemory)
+ {
+ *outMappedMemory = NULL;
+ }
+ return false;
+ }
- return oldWritePos;
+ if (streamOffset)
+ {
+ *streamOffset = mWritePosition;
+ }
+
+ mWritePosition += size;
+ return true;
}
bool IndexBufferInterface::unmapBuffer()
diff --git a/src/libGLESv2/renderer/IndexBuffer.h b/src/libGLESv2/renderer/IndexBuffer.h
index 98fa5fe..6fb885a 100644
--- a/src/libGLESv2/renderer/IndexBuffer.h
+++ b/src/libGLESv2/renderer/IndexBuffer.h
@@ -59,7 +59,7 @@
unsigned int getSerial() const;
- int mapBuffer(unsigned int size, void** outMappedMemory);
+ bool mapBuffer(unsigned int size, void** outMappedMemory, unsigned int *streamOffset);
bool unmapBuffer();
IndexBuffer *getIndexBuffer() const;
diff --git a/src/libGLESv2/renderer/IndexBuffer11.cpp b/src/libGLESv2/renderer/IndexBuffer11.cpp
index 2a442ec..66604c4 100644
--- a/src/libGLESv2/renderer/IndexBuffer11.cpp
+++ b/src/libGLESv2/renderer/IndexBuffer11.cpp
@@ -75,7 +75,8 @@
{
if (mBuffer)
{
- if (offset + size > mBufferSize)
+ // Check for integer overflows and out-out-bounds map requests
+ if (offset + size < offset || offset + size > mBufferSize)
{
ERR("Index buffer map range is not inside the buffer.");
return false;
diff --git a/src/libGLESv2/renderer/IndexDataManager.cpp b/src/libGLESv2/renderer/IndexDataManager.cpp
index 752d1fb..2eab550 100644
--- a/src/libGLESv2/renderer/IndexDataManager.cpp
+++ b/src/libGLESv2/renderer/IndexDataManager.cpp
@@ -132,7 +132,16 @@
default: UNREACHABLE(); alignedOffset = false;
}
- if (gl::ComputeTypeSize(type) * count + offset > static_cast<GLsizei>(storage->getSize()))
+ unsigned int typeSize = gl::ComputeTypeSize(type);
+
+ // check for integer overflows and underflows
+ if (static_cast<unsigned int>(offset) > (std::numeric_limits<unsigned int>::max() / typeSize) ||
+ static_cast<unsigned int>(count) > ((std::numeric_limits<unsigned int>::max() / typeSize) - offset))
+ {
+ return GL_OUT_OF_MEMORY;
+ }
+
+ if (typeSize * static_cast<unsigned int>(count) + offset > storage->getSize())
{
return GL_INVALID_OPERATION;
}
@@ -146,7 +155,7 @@
IndexBufferInterface *indexBuffer = streamingBuffer;
bool directStorage = alignedOffset && storage && storage->supportsDirectBinding() &&
destinationIndexType == type;
- UINT streamOffset = 0;
+ unsigned int streamOffset = 0;
if (directStorage)
{
@@ -176,7 +185,7 @@
}
else
{
- int convertCount = count;
+ unsigned int convertCount = count;
if (staticBuffer)
{
@@ -198,12 +207,22 @@
return GL_INVALID_OPERATION;
}
- unsigned int bufferSizeRequired = convertCount * gl::ComputeTypeSize(destinationIndexType);
- indexBuffer->reserveBufferSpace(bufferSizeRequired, type);
+ unsigned int indexTypeSize = gl::ComputeTypeSize(destinationIndexType);
+ if (convertCount > std::numeric_limits<unsigned int>::max() / indexTypeSize)
+ {
+ ERR("Reserving %u indicies of %u bytes each exceeds the maximum buffer size.", convertCount, indexTypeSize);
+ return GL_OUT_OF_MEMORY;
+ }
+
+ unsigned int bufferSizeRequired = convertCount * indexTypeSize;
+ if (!indexBuffer->reserveBufferSpace(bufferSizeRequired, type))
+ {
+ ERR("Failed to reserve %u bytes in an index buffer.", bufferSizeRequired);
+ return GL_OUT_OF_MEMORY;
+ }
void* output = NULL;
- streamOffset = indexBuffer->mapBuffer(bufferSizeRequired, &output);
- if (streamOffset == -1 || output == NULL)
+ if (!indexBuffer->mapBuffer(bufferSizeRequired, &output, &streamOffset))
{
ERR("Failed to map index buffer.");
return GL_OUT_OF_MEMORY;
@@ -254,7 +273,7 @@
mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_SHORT);
void* mappedMemory = NULL;
- if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+ if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
{
ERR("Failed to map counting buffer.");
return NULL;
@@ -284,7 +303,7 @@
mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT);
void* mappedMemory = NULL;
- if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+ if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
{
ERR("Failed to map counting buffer.");
return NULL;
diff --git a/src/libGLESv2/renderer/Renderer11.cpp b/src/libGLESv2/renderer/Renderer11.cpp
index 9173fc8..774cd71 100644
--- a/src/libGLESv2/renderer/Renderer11.cpp
+++ b/src/libGLESv2/renderer/Renderer11.cpp
@@ -1150,15 +1150,15 @@
}
void* mappedMemory = NULL;
- int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset;
+ if (!mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map index buffer for GL_LINE_LOOP.");
return gl::error(GL_OUT_OF_MEMORY);
}
unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
- unsigned int indexBufferOffset = static_cast<unsigned int>(offset);
+ unsigned int indexBufferOffset = offset;
switch (type)
{
@@ -1255,15 +1255,15 @@
}
void* mappedMemory = NULL;
- int offset = mTriangleFanIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset;
+ if (!mTriangleFanIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map scratch index buffer for GL_TRIANGLE_FAN.");
return gl::error(GL_OUT_OF_MEMORY);
}
unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
- unsigned int indexBufferOffset = static_cast<unsigned int>(offset);
+ unsigned int indexBufferOffset = offset;
switch (type)
{
diff --git a/src/libGLESv2/renderer/Renderer9.cpp b/src/libGLESv2/renderer/Renderer9.cpp
index 240367d..c6d0916 100644
--- a/src/libGLESv2/renderer/Renderer9.cpp
+++ b/src/libGLESv2/renderer/Renderer9.cpp
@@ -1454,7 +1454,7 @@
indices = static_cast<const GLubyte*>(storage->getData()) + offset;
}
- UINT startIndex = 0;
+ unsigned int startIndex = 0;
if (get32BitIndexSupport())
{
@@ -1488,14 +1488,14 @@
}
void* mappedMemory = NULL;
- int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset = 0;
+ if (!mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map index buffer for GL_LINE_LOOP.");
return gl::error(GL_OUT_OF_MEMORY);
}
- startIndex = static_cast<UINT>(offset) / 4;
+ startIndex = static_cast<unsigned int>(offset) / 4;
unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
switch (type)
@@ -1569,14 +1569,14 @@
}
void* mappedMemory = NULL;
- int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset;
+ if (mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map index buffer for GL_LINE_LOOP.");
return gl::error(GL_OUT_OF_MEMORY);
}
- startIndex = static_cast<UINT>(offset) / 2;
+ startIndex = static_cast<unsigned int>(offset) / 2;
unsigned short *data = reinterpret_cast<unsigned short*>(mappedMemory);
switch (type)
diff --git a/src/libGLESv2/renderer/VertexBuffer.cpp b/src/libGLESv2/renderer/VertexBuffer.cpp
index a858946..a073d95 100644
--- a/src/libGLESv2/renderer/VertexBuffer.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer.cpp
@@ -87,47 +87,76 @@
return mVertexBuffer->discard();
}
-int VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances)
+bool VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+ unsigned int *outStreamOffset)
{
+ unsigned int spaceRequired;
+ if (!mVertexBuffer->getSpaceRequired(attrib, count, instances, &spaceRequired))
+ {
+ return false;
+ }
+
+ if (mWritePosition + spaceRequired < mWritePosition)
+ {
+ return false;
+ }
+
if (!reserveSpace(mReservedSpace))
{
- return -1;
+ return false;
}
mReservedSpace = 0;
if (!mVertexBuffer->storeVertexAttributes(attrib, start, count, instances, mWritePosition))
{
- return -1;
+ return false;
}
- int oldWritePos = static_cast<int>(mWritePosition);
- mWritePosition += mVertexBuffer->getSpaceRequired(attrib, count, instances);
+ if (outStreamOffset)
+ {
+ *outStreamOffset = mWritePosition;
+ }
- return oldWritePos;
+ mWritePosition += spaceRequired;
+
+ return true;
}
-int VertexBufferInterface::storeRawData(const void* data, unsigned int size)
+bool VertexBufferInterface::storeRawData(const void* data, unsigned int size, unsigned int *outStreamOffset)
{
+ if (mWritePosition + size < mWritePosition)
+ {
+ return false;
+ }
+
if (!reserveSpace(mReservedSpace))
{
- return -1;
+ return false;
}
mReservedSpace = 0;
if (!mVertexBuffer->storeRawData(data, size, mWritePosition))
{
- return -1;
+ return false;
}
- int oldWritePos = static_cast<int>(mWritePosition);
+ if (outStreamOffset)
+ {
+ *outStreamOffset = mWritePosition;
+ }
+
mWritePosition += size;
- return oldWritePos;
+ return true;
}
bool VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
{
- unsigned int requiredSpace = mVertexBuffer->getSpaceRequired(attribute, count, instances);
+ unsigned int requiredSpace;
+ if (!mVertexBuffer->getSpaceRequired(attribute, count, instances, &requiredSpace))
+ {
+ return false;
+ }
// Protect against integer overflow
if (mReservedSpace + requiredSpace < mReservedSpace)
@@ -195,7 +224,7 @@
{
}
-int StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attribute)
+bool StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attribute, unsigned int *outStreamOffset)
{
for (unsigned int element = 0; element < mCache.size(); element++)
{
@@ -206,12 +235,16 @@
{
if (mCache[element].attributeOffset == attribute.mOffset % attribute.stride())
{
- return mCache[element].streamOffset;
+ if (outStreamOffset)
+ {
+ *outStreamOffset = mCache[element].streamOffset;
+ }
+ return true;
}
}
}
- return -1;
+ return false;
}
bool StaticVertexBufferInterface::reserveSpace(unsigned int size)
@@ -233,13 +266,27 @@
}
}
-int StaticVertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances)
+bool StaticVertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+ unsigned int *outStreamOffset)
{
- int attributeOffset = attrib.mOffset % attrib.stride();
- VertexElement element = { attrib.mType, attrib.mSize, attrib.stride(), attrib.mNormalized, attributeOffset, getWritePosition() };
- mCache.push_back(element);
+ unsigned int streamOffset;
+ if (VertexBufferInterface::storeVertexAttributes(attrib, start, count, instances, &streamOffset))
+ {
+ int attributeOffset = attrib.mOffset % attrib.stride();
+ VertexElement element = { attrib.mType, attrib.mSize, attrib.stride(), attrib.mNormalized, attributeOffset, streamOffset };
+ mCache.push_back(element);
- return VertexBufferInterface::storeVertexAttributes(attrib, start, count, instances);
+ if (outStreamOffset)
+ {
+ *outStreamOffset = streamOffset;
+ }
+
+ return true;
+ }
+ else
+ {
+ return false;
+ }
}
}
diff --git a/src/libGLESv2/renderer/VertexBuffer.h b/src/libGLESv2/renderer/VertexBuffer.h
index c474b05..cbafdd2 100644
--- a/src/libGLESv2/renderer/VertexBuffer.h
+++ b/src/libGLESv2/renderer/VertexBuffer.h
@@ -33,8 +33,8 @@
GLsizei instances, unsigned int offset) = 0;
virtual bool storeRawData(const void* data, unsigned int size, unsigned int offset) = 0;
- virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
- GLsizei instances) const = 0;
+ virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+ unsigned int *outSpaceRequired) const = 0;
virtual bool requiresConversion(const gl::VertexAttribute &attrib) const = 0;
@@ -67,8 +67,9 @@
unsigned int getSerial() const;
- virtual int storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances);
- virtual int storeRawData(const void* data, unsigned int size);
+ virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+ unsigned int *outStreamOffset);
+ virtual bool storeRawData(const void* data, unsigned int size, unsigned int *outStreamOffset);
VertexBuffer* getVertexBuffer() const;
@@ -110,10 +111,10 @@
explicit StaticVertexBufferInterface(rx::Renderer *renderer);
~StaticVertexBufferInterface();
- int storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances);
+ bool storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
+ unsigned int *outStreamOffset);
- // Returns the offset into the vertex buffer, or -1 if not found
- int lookupAttribute(const gl::VertexAttribute &attribute);
+ bool lookupAttribute(const gl::VertexAttribute &attribute, unsigned int* outStreamOffset);
protected:
bool reserveSpace(unsigned int size);
diff --git a/src/libGLESv2/renderer/VertexBuffer11.cpp b/src/libGLESv2/renderer/VertexBuffer11.cpp
index 92c8755..521da80 100644
--- a/src/libGLESv2/renderer/VertexBuffer11.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer11.cpp
@@ -152,18 +152,40 @@
}
}
-unsigned int VertexBuffer11::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
- GLsizei instances) const
+bool VertexBuffer11::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
+ GLsizei instances, unsigned int *outSpaceRequired) const
{
unsigned int elementSize = getVertexConversion(attrib).outputElementSize;
+ unsigned int elementCount = 0;
if (instances == 0 || attrib.mDivisor == 0)
{
- return elementSize * count;
+ elementCount = count;
}
else
{
- return elementSize * ((instances + attrib.mDivisor - 1) / attrib.mDivisor);
+ if (static_cast<unsigned int>(instances) < std::numeric_limits<unsigned int>::max() - (attrib.mDivisor - 1))
+ {
+ // Round up
+ elementCount = (static_cast<unsigned int>(instances) + (attrib.mDivisor - 1)) / attrib.mDivisor;
+ }
+ else
+ {
+ elementCount = instances / attrib.mDivisor;
+ }
+ }
+
+ if (elementSize <= std::numeric_limits<unsigned int>::max() / elementCount)
+ {
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * elementCount;
+ }
+ return true;
+ }
+ else
+ {
+ return false;
}
}
diff --git a/src/libGLESv2/renderer/VertexBuffer11.h b/src/libGLESv2/renderer/VertexBuffer11.h
index 75e0250..eceb426 100644
--- a/src/libGLESv2/renderer/VertexBuffer11.h
+++ b/src/libGLESv2/renderer/VertexBuffer11.h
@@ -26,10 +26,11 @@
static VertexBuffer11 *makeVertexBuffer11(VertexBuffer *vetexBuffer);
virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, GLint start, GLsizei count, GLsizei instances,
- unsigned int offset);
+ unsigned int offset);
virtual bool storeRawData(const void* data, unsigned int size, unsigned int offset);
- virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const;
+ virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+ unsigned int *outSpaceRequired) const;
virtual bool requiresConversion(const gl::VertexAttribute &attrib) const;
@@ -70,4 +71,4 @@
}
-#endif // LIBGLESV2_RENDERER_VERTEXBUFFER11_H_
\ No newline at end of file
+#endif // LIBGLESV2_RENDERER_VERTEXBUFFER11_H_
diff --git a/src/libGLESv2/renderer/VertexBuffer9.cpp b/src/libGLESv2/renderer/VertexBuffer9.cpp
index 76dc73e..534be25 100644
--- a/src/libGLESv2/renderer/VertexBuffer9.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer9.cpp
@@ -95,7 +95,14 @@
DWORD lockFlags = mDynamicUsage ? D3DLOCK_NOOVERWRITE : 0;
void *mapPtr = NULL;
- HRESULT result = mVertexBuffer->Lock(offset, spaceRequired(attrib, count, instances), &mapPtr, lockFlags);
+
+ unsigned int mapSize;
+ if (!spaceRequired(attrib, count, instances, &mapSize))
+ {
+ return false;
+ }
+
+ HRESULT result = mVertexBuffer->Lock(offset, mapSize, &mapPtr, lockFlags);
if (FAILED(result))
{
@@ -167,9 +174,10 @@
}
}
-unsigned int VertexBuffer9::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const
+bool VertexBuffer9::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+ unsigned int *outSpaceRequired) const
{
- return spaceRequired(attrib, count, instances);
+ return spaceRequired(attrib, count, instances, outSpaceRequired);
}
bool VertexBuffer9::requiresConversion(const gl::VertexAttribute &attrib) const
@@ -179,7 +187,8 @@
unsigned int VertexBuffer9::getVertexSize(const gl::VertexAttribute &attrib) const
{
- return spaceRequired(attrib, 1, 0);
+ unsigned int spaceRequired;
+ return getSpaceRequired(attrib, 1, 0, &spaceRequired) ? spaceRequired : 0;
}
D3DDECLTYPE VertexBuffer9::getDeclType(const gl::VertexAttribute &attrib) const
@@ -469,17 +478,52 @@
return mFormatConverters[typeIndex(attribute.mType)][attribute.mNormalized][attribute.mSize - 1];
}
-unsigned int VertexBuffer9::spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances)
+bool VertexBuffer9::spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances,
+ unsigned int *outSpaceRequired)
{
unsigned int elementSize = formatConverter(attrib).outputElementSize;
if (instances == 0 || attrib.mDivisor == 0)
{
- return elementSize * count;
+ unsigned int elementCount = 0;
+ if (instances == 0 || attrib.mDivisor == 0)
+ {
+ elementCount = count;
+ }
+ else
+ {
+ if (static_cast<unsigned int>(instances) < std::numeric_limits<unsigned int>::max() - (attrib.mDivisor - 1))
+ {
+ // Round up
+ elementCount = (static_cast<unsigned int>(instances) + (attrib.mDivisor - 1)) / attrib.mDivisor;
+ }
+ else
+ {
+ elementCount = static_cast<unsigned int>(instances) / attrib.mDivisor;
+ }
+ }
+
+ if (elementSize <= std::numeric_limits<unsigned int>::max() / elementCount)
+ {
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * elementCount;
+ }
+ return true;
+ }
+ else
+ {
+ return false;
+ }
}
else
{
- return elementSize * ((instances + attrib.mDivisor - 1) / attrib.mDivisor);
+ const unsigned int elementSize = 4;
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * 4;
+ }
+ return true;
}
}
diff --git a/src/libGLESv2/renderer/VertexBuffer9.h b/src/libGLESv2/renderer/VertexBuffer9.h
index f771635..2f88117 100644
--- a/src/libGLESv2/renderer/VertexBuffer9.h
+++ b/src/libGLESv2/renderer/VertexBuffer9.h
@@ -29,7 +29,7 @@
unsigned int offset);
virtual bool storeRawData(const void* data, unsigned int size, unsigned int offset);
- virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const;
+ virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances, unsigned int *outSpaceRequired) const;
virtual bool requiresConversion(const gl::VertexAttribute &attrib) const;
@@ -82,7 +82,8 @@
static unsigned int typeIndex(GLenum type);
static const FormatConverter &formatConverter(const gl::VertexAttribute &attribute);
- static unsigned int spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances);
+ static bool spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances,
+ unsigned int *outSpaceRequired);
};
}
diff --git a/src/libGLESv2/renderer/VertexDataManager.cpp b/src/libGLESv2/renderer/VertexDataManager.cpp
index fd869b1..8790463 100644
--- a/src/libGLESv2/renderer/VertexDataManager.cpp
+++ b/src/libGLESv2/renderer/VertexDataManager.cpp
@@ -26,9 +26,15 @@
namespace rx
{
-static int elementsInBuffer(const gl::VertexAttribute &attribute, int size)
+static int elementsInBuffer(const gl::VertexAttribute &attribute, unsigned int size)
{
- int stride = attribute.stride();
+ // Size cannot be larger than a GLsizei
+ if (size > static_cast<unsigned int>(std::numeric_limits<int>::max()))
+ {
+ size = static_cast<unsigned int>(std::numeric_limits<int>::max());
+ }
+
+ GLsizei stride = attribute.stride();
return (size - attribute.mOffset % stride + (stride - attribute.typeSize())) / stride;
}
@@ -92,7 +98,7 @@
gl::Buffer *buffer = attribs[i].mBoundBuffer.get();
StaticVertexBufferInterface *staticBuffer = buffer ? buffer->getStaticVertexBuffer() : NULL;
- if (staticBuffer && staticBuffer->getBufferSize() > 0 && staticBuffer->lookupAttribute(attribs[i]) == -1 &&
+ if (staticBuffer && staticBuffer->getBufferSize() > 0 && !staticBuffer->lookupAttribute(attribs[i], NULL) &&
!directStoragePossible(staticBuffer, attribs[i]))
{
buffer->invalidateStaticData();
@@ -155,7 +161,7 @@
BufferStorage *storage = buffer ? buffer->getStorage() : NULL;
bool directStorage = directStoragePossible(vertexBuffer, attribs[i]);
- std::size_t streamOffset = -1;
+ unsigned int streamOffset = 0;
unsigned int outputElementSize = 0;
if (directStorage)
@@ -166,37 +172,39 @@
}
else if (staticBuffer)
{
- streamOffset = staticBuffer->lookupAttribute(attribs[i]);
- outputElementSize = staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
+ if (!staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize))
+ {
+ return GL_OUT_OF_MEMORY;
+ }
- if (streamOffset == -1)
+ if (!staticBuffer->lookupAttribute(attribs[i], &streamOffset))
{
// Convert the entire buffer
int totalCount = elementsInBuffer(attribs[i], storage->getSize());
int startIndex = attribs[i].mOffset / attribs[i].stride();
- streamOffset = staticBuffer->storeVertexAttributes(attribs[i], -startIndex, totalCount, 0);
- }
-
- if (streamOffset != -1)
- {
- streamOffset += (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
-
- if (instances == 0 || attribs[i].mDivisor == 0)
+ if (!staticBuffer->storeVertexAttributes(attribs[i], -startIndex, totalCount, 0, &streamOffset))
{
- streamOffset += start * outputElementSize;
+ return GL_OUT_OF_MEMORY;
}
}
+
+ unsigned int firstElementOffset = (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
+ unsigned int startOffset = (instances == 0 || attribs[i].mDivisor == 0) ? start * outputElementSize : 0;
+ if (streamOffset + firstElementOffset + startOffset < streamOffset)
+ {
+ return GL_OUT_OF_MEMORY;
+ }
+
+ streamOffset += firstElementOffset + startOffset;
}
else
{
- outputElementSize = mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
- streamOffset = mStreamingBuffer->storeVertexAttributes(attribs[i], start, count, instances);
- }
-
- if (streamOffset == -1)
- {
- return GL_OUT_OF_MEMORY;
+ if (!mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize) ||
+ !mStreamingBuffer->storeVertexAttributes(attribs[i], start, count, instances, &streamOffset))
+ {
+ return GL_OUT_OF_MEMORY;
+ }
}
translated[i].storage = directStorage ? storage : NULL;
@@ -227,8 +235,9 @@
{
return GL_OUT_OF_MEMORY;
}
- int streamOffset = buffer->storeRawData(attribs[i].mCurrentValue, requiredSpace);
- if (streamOffset == -1)
+
+ unsigned int streamOffset;
+ if (!buffer->storeRawData(attribs[i].mCurrentValue, requiredSpace, &streamOffset))
{
return GL_OUT_OF_MEMORY;
}
diff --git a/src/libGLESv2/renderer/VertexDataManager.h b/src/libGLESv2/renderer/VertexDataManager.h
index 28387e6..1a87865 100644
--- a/src/libGLESv2/renderer/VertexDataManager.h
+++ b/src/libGLESv2/renderer/VertexDataManager.h
@@ -31,8 +31,8 @@
bool active;
const gl::VertexAttribute *attribute;
- UINT offset;
- UINT stride; // 0 means not to advance the read pointer at all
+ unsigned int offset;
+ unsigned int stride; // 0 means not to advance the read pointer at all
VertexBuffer *vertexBuffer;
BufferStorage *storage;
