PyBullet OpenGL/EGL hardware getCameraImage: use glViewport to reduce the glReadPixels calling cost dramatically for small images

PyBullet Allow OpenGL/EGL hardware to render segmentation mask. Use pybullet.ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX or pybullet.ER_SEGMENTATION_MASK
PyBullet.removeBody fix indexing bug (use foundIndex, not i)
PyBullet bump up version to 2.2.3

build3/stringifyShaders.bat

@@ -2,6 +2,12 @@
 rem @echo off
 
 
+
+
+premake4 --file=stringifyKernel.lua --kernelfile="../examples/OpenGLWindow/Shaders/segmentationMaskInstancingVS.glsl" --headerfile="../examples/OpenGLWindow/Shaders/segmentationMaskInstancingVS.h" --stringname="segmentationMaskInstancingVertexShader" stringify
+premake4 --file=stringifyKernel.lua --kernelfile="../examples/OpenGLWindow/Shaders/segmentationMaskInstancingPS.glsl" --headerfile="../examples/OpenGLWindow/Shaders/segmentationMaskInstancingPS.h" --stringname="segmentationMaskInstancingFragmentShader" stringify
+
+
 premake4 --file=stringifyKernel.lua --kernelfile="../examples/OpenGLWindow/Shaders/instancingVS.glsl" --headerfile="../examples/OpenGLWindow/Shaders/instancingVS.h" --stringname="instancingVertexShader" stringify
 premake4 --file=stringifyKernel.lua --kernelfile="../examples/OpenGLWindow/Shaders/instancingPS.glsl" --headerfile="../examples/OpenGLWindow/Shaders/instancingPS.h" --stringname="instancingFragmentShader" stringify
 premake4 --file=stringifyKernel.lua --kernelfile="../examples/OpenGLWindow/Shaders/pointSpriteVS.glsl" --headerfile="../examples/OpenGLWindow/Shaders/pointSpriteVS.h" --stringname="pointSpriteVertexShader" stringify

examples/CommonInterfaces/CommonGraphicsAppInterface.h

@@ -82,6 +82,7 @@ struct CommonGraphicsApp
 	virtual void dumpFramesToVideo(const char* mp4Filename) {}
 
 	virtual void getScreenPixels(unsigned char* rgbaBuffer, int bufferSizeInBytes, float* depthBuffer, int depthBufferSizeInBytes) {}
+	virtual void setViewport(int width, int height) {}
 
 	virtual void getBackgroundColor(float* red, float* green, float* blue) const
 	{

examples/CommonInterfaces/CommonRenderInterface.h

@@ -18,6 +18,7 @@ enum
 	B3_USE_SHADOWMAP_RENDERMODE_REFLECTION,
 	B3_USE_SHADOWMAP_RENDERMODE_REFLECTION_PLANE,
 	B3_USE_PROJECTIVE_TEXTURE_RENDERMODE,
+	B3_SEGMENTATION_MASK_RENDERMODE,
 };
 
 struct GfxVertexFormat0

examples/ExampleBrowser/OpenGLGuiHelper.cpp

@@ -202,6 +202,7 @@ struct OpenGLGuiHelperInternalData
 
 	btAlignedObjectArray<unsigned char> m_rgbaPixelBuffer1;
 	btAlignedObjectArray<float> m_depthBuffer1;
+	btAlignedObjectArray<int> m_segmentationMaskBuffer;
 	btHashMap<MyHashShape, int> m_hashShapes;
 
 	VisualizerFlagCallback m_visualizerFlagCallback;
@@ -990,7 +991,7 @@ void OpenGLGuiHelper::setVisualizerFlag(int flag, int enable)
 		getRenderInterface()->setPlaneReflectionShapeIndex(enable);
 	}
 	if (m_data->m_visualizerFlagCallback)
-		(m_data->m_visualizerFlagCallback)(flag, enable);
+		(m_data->m_visualizerFlagCallback)(flag, enable!=0);
 }
 
 void OpenGLGuiHelper::resetCamera(float camDist, float yaw, float pitch, float camPosX, float camPosY, float camPosZ)
@@ -1070,6 +1071,7 @@ void OpenGLGuiHelper::setProjectiveTexture(bool useProjectiveTexture)
 	m_data->m_glApp->m_renderer->setProjectiveTexture(useProjectiveTexture);
 }
 
+
 void OpenGLGuiHelper::copyCameraImageData(const float viewMatrix[16], const float projectionMatrix[16],
 										  unsigned char* pixelsRGBA, int rgbaBufferSizeInPixels,
 										  float* depthBuffer, int depthBufferSizeInPixels,
@@ -1077,8 +1079,11 @@ void OpenGLGuiHelper::copyCameraImageData(const float viewMatrix[16], const floa
 										  int startPixelIndex, int destinationWidth,
 										  int destinationHeight, int* numPixelsCopied)
 {
-	int sourceWidth = m_data->m_glApp->m_window->getWidth() * m_data->m_glApp->m_window->getRetinaScale();
+	
-	int sourceHeight = m_data->m_glApp->m_window->getHeight() * m_data->m_glApp->m_window->getRetinaScale();
+	
+	int sourceWidth = btMin(destinationWidth, (int)(m_data->m_glApp->m_window->getWidth() * m_data->m_glApp->m_window->getRetinaScale()));
+	int sourceHeight = btMin(destinationHeight, (int)(m_data->m_glApp->m_window->getHeight() * m_data->m_glApp->m_window->getRetinaScale()));
+	m_data->m_glApp->setViewport(sourceWidth, sourceHeight);
 
 	if (numPixelsCopied)
 		*numPixelsCopied = 0;
@@ -1099,8 +1104,7 @@ void OpenGLGuiHelper::copyCameraImageData(const float viewMatrix[16], const floa
 				BT_PROFILE("renderScene");
 				getRenderInterface()->renderScene();
 			}
-			getRenderInterface()->setActiveCamera(oldCam);
+			
-
 			{
 				BT_PROFILE("copy pixels");
 				btAlignedObjectArray<unsigned char> sourceRgbaPixelBuffer;
@@ -1151,6 +1155,65 @@ void OpenGLGuiHelper::copyCameraImageData(const float viewMatrix[16], const floa
 				}
 			}
 
+			//segmentation mask
+
+			if (segmentationMaskBuffer)
+			{
+				{
+					m_data->m_glApp->m_window->startRendering();
+					BT_PROFILE("renderScene");
+					getRenderInterface()->renderSceneInternal(B3_SEGMENTATION_MASK_RENDERMODE);
+				}
+			
+				{
+					BT_PROFILE("copy pixels");
+					btAlignedObjectArray<unsigned char> sourceRgbaPixelBuffer;
+					btAlignedObjectArray<float> sourceDepthBuffer;
+					//copy the image into our local cache
+					sourceRgbaPixelBuffer.resize(sourceWidth * sourceHeight * numBytesPerPixel);
+					sourceDepthBuffer.resize(sourceWidth * sourceHeight);
+					{
+						BT_PROFILE("getScreenPixelsSegmentationMask");
+						m_data->m_glApp->getScreenPixels(&(sourceRgbaPixelBuffer[0]), sourceRgbaPixelBuffer.size(), &sourceDepthBuffer[0], sizeof(float) * sourceDepthBuffer.size());
+					}
+					m_data->m_segmentationMaskBuffer.resize(destinationWidth * destinationHeight,-1);
+
+					//rescale and flip
+					{
+						BT_PROFILE("resize and flip segmentation mask");
+						for (int j = 0; j < destinationHeight; j++)
+						{
+							for (int i = 0; i < destinationWidth; i++)
+							{
+								int xIndex = int(float(i) * (float(sourceWidth) / float(destinationWidth)));
+								int yIndex = int(float(destinationHeight - 1 - j) * (float(sourceHeight) / float(destinationHeight)));
+								btClamp(xIndex, 0, sourceWidth);
+								btClamp(yIndex, 0, sourceHeight);
+								int bytesPerPixel = 4;  //RGBA
+								int sourcePixelIndex = (xIndex + yIndex * sourceWidth) * bytesPerPixel;
+								int sourceDepthIndex = xIndex + yIndex * sourceWidth;
+
+								if (segmentationMaskBuffer)
+								{
+									float depth = sourceDepthBuffer[sourceDepthIndex];
+									if (depth<1)
+									{
+										int segMask = sourceRgbaPixelBuffer[sourcePixelIndex + 0]+256*(sourceRgbaPixelBuffer[sourcePixelIndex + 1])+256*256*(sourceRgbaPixelBuffer[sourcePixelIndex + 2]);
+										m_data->m_segmentationMaskBuffer[i + j * destinationWidth] = segMask;
+									} else
+									{
+										m_data->m_segmentationMaskBuffer[i + j * destinationWidth] = -1;
+									}
+								}
+							}
+						}
+					}
+				}
+			}
+
+
+			getRenderInterface()->setActiveCamera(oldCam);
+
 			if (1)
 			{
 				getRenderInterface()->getActiveCamera()->disableVRCamera();
@@ -1159,6 +1222,8 @@ void OpenGLGuiHelper::copyCameraImageData(const float viewMatrix[16], const floa
 				getRenderInterface()->updateCamera(dg.upAxis);
 				m_data->m_glApp->m_window->startRendering();
 			}
+			
+
 		}
 		if (pixelsRGBA)
 		{
@@ -1178,9 +1243,19 @@ void OpenGLGuiHelper::copyCameraImageData(const float viewMatrix[16], const floa
 				depthBuffer[i] = m_data->m_depthBuffer1[i + startPixelIndex];
 			}
 		}
+		if (segmentationMaskBuffer)
+		{
+			BT_PROFILE("copy segmentation mask pixels");
+			for (int i = 0; i < numRequestedPixels; i++)
+			{
+				segmentationMaskBuffer[i] = m_data->m_segmentationMaskBuffer[i + startPixelIndex];
+			}
+		}
 		if (numPixelsCopied)
 			*numPixelsCopied = numRequestedPixels;
 	}
+		
+	m_data->m_glApp->setViewport(-1,-1);
 }
 
 struct MyConvertPointerSizeT
@@ -1271,10 +1346,10 @@ void OpenGLGuiHelper::computeSoftBodyVertices(btCollisionShape* collisionShape,
 	b3Assert(collisionShape->getUserPointer());
 	btSoftBody* psb = (btSoftBody*)collisionShape->getUserPointer();
 	gfxVertices.resize(psb->m_faces.size() * 3);
-	int i, j, k;
+	
-	for (i = 0; i < psb->m_faces.size(); i++)  // Foreach face
+	for (int i = 0; i < psb->m_faces.size(); i++)  // Foreach face
 	{
-		for (k = 0; k < 3; k++)  // Foreach vertex on a face
+		for (int k = 0; k < 3; k++)  // Foreach vertex on a face
 		{
 			int currentIndex = i * 3 + k;
 			for (int j = 0; j < 3; j++)

examples/OpenGLWindow/GLInstancingRenderer.cpp

@@ -19,6 +19,25 @@ bool useShadowMap = true;  // true;//false;//true;
 int shadowMapWidth = 4096;
 int shadowMapHeight = 4096;
 float shadowMapWorldSize = 10;
+#include <stdio.h>
+
+struct caster2
+{
+	void setInt(int v)
+	{
+		i = v;
+	}
+	float getFloat()
+	{
+		float v = ((float)i) + .25;
+		return v;
+	}
+
+	union {
+		int i;
+		float f;
+	};
+};
 
 #define MAX_POINTS_IN_BATCH 1024
 #define MAX_LINES_IN_BATCH 1024
@@ -75,6 +94,10 @@ float shadowMapWorldSize = 10;
 #include "Shaders/useShadowMapInstancingPS.h"
 #include "Shaders/projectiveTextureInstancingVS.h"
 #include "Shaders/projectiveTextureInstancingPS.h"
+
+#include "Shaders/segmentationMaskInstancingVS.h"
+#include "Shaders/segmentationMaskInstancingPS.h"
+
 #include "Shaders/linesPS.h"
 #include "Shaders/linesVS.h"
 
@@ -261,8 +284,10 @@ static GLuint linesShader;                        // The line renderer
 static GLuint useShadowMapInstancingShader;       // The shadow instancing renderer
 static GLuint createShadowMapInstancingShader;    // The shadow instancing renderer
 static GLuint projectiveTextureInstancingShader;  // The projective texture instancing renderer
-static GLuint instancingShader;                   // The instancing renderer
+static GLuint segmentationMaskInstancingShader;   // The segmentation mask instancing renderer
-static GLuint instancingShaderPointSprite;        // The point sprite instancing renderer
+
+static GLuint instancingShader;             // The instancing renderer
+static GLuint instancingShaderPointSprite;  // The point sprite instancing renderer
 
 //static bool                 done = false;
 
@@ -312,6 +337,9 @@ static GLint ModelViewMatrix = 0;
 static GLint ProjectionMatrix = 0;
 static GLint regularLightDirIn = 0;
 
+static GLint segmentationMaskModelViewMatrix = 0;
+static GLint segmentationMaskProjectionMatrix = 0;
+
 static GLint uniform_texture_diffuse = 0;
 
 static GLint screenWidthPointSprite = 0;
@@ -338,7 +366,7 @@ GLInstancingRenderer::GLInstancingRenderer(int maxNumObjectCapacity, int maxShap
 	m_data->m_instance_positions_ptr.resize(m_data->m_maxNumObjectCapacity * 4);
 	m_data->m_instance_quaternion_ptr.resize(m_data->m_maxNumObjectCapacity * 4);
 	m_data->m_instance_colors_ptr.resize(m_data->m_maxNumObjectCapacity * 4);
-	m_data->m_instance_scale_ptr.resize(m_data->m_maxNumObjectCapacity * 3);
+	m_data->m_instance_scale_ptr.resize(m_data->m_maxNumObjectCapacity * 4);
 }
 
 void GLInstancingRenderer::removeAllInstances()
@@ -506,9 +534,12 @@ void GLInstancingRenderer::writeSingleInstanceScaleToCPU(const float* scale, int
 	b3Assert(pg);
 	int srcIndex = pg->m_internalInstanceIndex;
 
-	m_data->m_instance_scale_ptr[srcIndex * 3 + 0] = scale[0];
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 0] = scale[0];
-	m_data->m_instance_scale_ptr[srcIndex * 3 + 1] = scale[1];
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 1] = scale[1];
-	m_data->m_instance_scale_ptr[srcIndex * 3 + 2] = scale[2];
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 2] = scale[2];
+	caster2 c;
+	c.setInt(srcIndex2);
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 3] = c.getFloat();
 }
 
 void GLInstancingRenderer::writeSingleInstanceSpecularColorToCPU(const double* specular, int srcIndex2)
@@ -570,9 +601,12 @@ void GLInstancingRenderer::writeSingleInstanceScaleToCPU(const double* scale, in
 	b3Assert(pg);
 	int srcIndex = pg->m_internalInstanceIndex;
 
-	m_data->m_instance_scale_ptr[srcIndex * 3 + 0] = scale[0];
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 0] = scale[0];
-	m_data->m_instance_scale_ptr[srcIndex * 3 + 1] = scale[1];
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 1] = scale[1];
-	m_data->m_instance_scale_ptr[srcIndex * 3 + 2] = scale[2];
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 2] = scale[2];
+	caster2 c;
+	c.setInt(srcIndex2);
+	m_data->m_instance_scale_ptr[srcIndex * 4 + 3] = c.getFloat();
 }
 
 void GLInstancingRenderer::writeSingleInstanceTransformToGPU(float* position, float* orientation, int objectUniqueId)
@@ -675,7 +709,7 @@ void GLInstancingRenderer::writeTransforms()
 		}
 		{
 			//			B3_PROFILE("glBufferSubData scale");
-			glBufferSubData(GL_ARRAY_BUFFER, m_data->m_maxShapeCapacityInBytes + POSITION_BUFFER_SIZE + ORIENTATION_BUFFER_SIZE + COLOR_BUFFER_SIZE, m_data->m_totalNumInstances * sizeof(float) * 3,
+			glBufferSubData(GL_ARRAY_BUFFER, m_data->m_maxShapeCapacityInBytes + POSITION_BUFFER_SIZE + ORIENTATION_BUFFER_SIZE + COLOR_BUFFER_SIZE, m_data->m_totalNumInstances * sizeof(float) * 4,
 							&m_data->m_instance_scale_ptr[0]);
 		}
 	}
@@ -718,9 +752,10 @@ void GLInstancingRenderer::writeTransforms()
 				colors[srcIndex * 4 + 2] = m_data->m_instance_colors_ptr[srcIndex * 4 + 2];
 				colors[srcIndex * 4 + 3] = m_data->m_instance_colors_ptr[srcIndex * 4 + 3];
 
-				scaling[srcIndex * 3] = m_data->m_instance_scale_ptr[srcIndex * 3];
+				scaling[srcIndex * 4] = m_data->m_instance_scale_ptr[srcIndex * 4];
-				scaling[srcIndex * 3 + 1] = m_data->m_instance_scale_ptr[srcIndex * 3 + 1];
+				scaling[srcIndex * 4 + 1] = m_data->m_instance_scale_ptr[srcIndex * 4 + 1];
-				scaling[srcIndex * 3 + 2] = m_data->m_instance_scale_ptr[srcIndex * 3 + 2];
+				scaling[srcIndex * 4 + 2] = m_data->m_instance_scale_ptr[srcIndex * 4 + 2];
+				scaling[srcIndex * 4 + 3] = m_data->m_instance_scale_ptr[srcIndex * 4 + 3];
 			}
 		}
 	}
@@ -782,9 +817,10 @@ void GLInstancingRenderer::rebuildGraphicsInstances()
 		pg->m_color[1] = m_data->m_instance_colors_ptr[srcIndex * 4 + 1];
 		pg->m_color[2] = m_data->m_instance_colors_ptr[srcIndex * 4 + 2];
 		pg->m_color[3] = m_data->m_instance_colors_ptr[srcIndex * 4 + 3];
-		pg->m_scale[0] = m_data->m_instance_scale_ptr[srcIndex * 3 + 0];
+		pg->m_scale[0] = m_data->m_instance_scale_ptr[srcIndex * 4 + 0];
-		pg->m_scale[1] = m_data->m_instance_scale_ptr[srcIndex * 3 + 1];
+		pg->m_scale[1] = m_data->m_instance_scale_ptr[srcIndex * 4 + 1];
-		pg->m_scale[2] = m_data->m_instance_scale_ptr[srcIndex * 3 + 2];
+		pg->m_scale[2] = m_data->m_instance_scale_ptr[srcIndex * 4 + 2];
+		pg->m_scale[3] = m_data->m_instance_scale_ptr[srcIndex * 4 + 3];
 	}
 	for (int i = 0; i < m_graphicsInstances.size(); i++)
 	{
@@ -861,9 +897,12 @@ int GLInstancingRenderer::registerGraphicsInstanceInternal(int newUid, const flo
 		m_data->m_instance_colors_ptr[index * 4 + 2] = color[2];
 		m_data->m_instance_colors_ptr[index * 4 + 3] = color[3];
 
-		m_data->m_instance_scale_ptr[index * 3] = scaling[0];
+		m_data->m_instance_scale_ptr[index * 4] = scaling[0];
-		m_data->m_instance_scale_ptr[index * 3 + 1] = scaling[1];
+		m_data->m_instance_scale_ptr[index * 4 + 1] = scaling[1];
-		m_data->m_instance_scale_ptr[index * 3 + 2] = scaling[2];
+		m_data->m_instance_scale_ptr[index * 4 + 2] = scaling[2];
+		caster2 c;
+		c.setInt(newUid);
+		m_data->m_instance_scale_ptr[index * 4 + 3] = c.getFloat();
 
 		if (color[3] < 1 && color[3] > 0)
 		{
@@ -912,9 +951,12 @@ int GLInstancingRenderer::registerGraphicsInstance(int shapeIndex, const float*
 		m_data->m_instance_colors_ptr[srcIndex * 4 + 2] = color[2];
 		m_data->m_instance_colors_ptr[srcIndex * 4 + 3] = color[3];
 
-		m_data->m_instance_scale_ptr[srcIndex * 3 + 0] = scaling[0];
+		m_data->m_instance_scale_ptr[srcIndex * 4 + 0] = scaling[0];
-		m_data->m_instance_scale_ptr[srcIndex * 3 + 1] = scaling[1];
+		m_data->m_instance_scale_ptr[srcIndex * 4 + 1] = scaling[1];
-		m_data->m_instance_scale_ptr[srcIndex * 3 + 2] = scaling[2];
+		m_data->m_instance_scale_ptr[srcIndex * 4 + 2] = scaling[2];
+		caster2 c;
+		c.setInt(newUid);
+		m_data->m_instance_scale_ptr[srcIndex * 4 + 3] = c.getFloat();
 
 		rebuildGraphicsInstances();
 	}
@@ -1124,7 +1166,7 @@ void GLInstancingRenderer::InitShaders()
 	int POSITION_BUFFER_SIZE = (m_data->m_maxNumObjectCapacity * sizeof(float) * 4);
 	int ORIENTATION_BUFFER_SIZE = (m_data->m_maxNumObjectCapacity * sizeof(float) * 4);
 	int COLOR_BUFFER_SIZE = (m_data->m_maxNumObjectCapacity * sizeof(float) * 4);
-	int SCALE_BUFFER_SIZE = (m_data->m_maxNumObjectCapacity * sizeof(float) * 3);
+	int SCALE_BUFFER_SIZE = (m_data->m_maxNumObjectCapacity * sizeof(float) * 4);
 
 	{
 		triangleShaderProgram = gltLoadShaderPair(triangleVertexShaderText, triangleFragmentShader);
@@ -1235,6 +1277,15 @@ void GLInstancingRenderer::InitShaders()
 
 	glUseProgram(0);
 
+	segmentationMaskInstancingShader = gltLoadShaderPair(segmentationMaskInstancingVertexShader, segmentationMaskInstancingFragmentShader);
+	glLinkProgram(segmentationMaskInstancingShader);
+	glUseProgram(segmentationMaskInstancingShader);
+
+	segmentationMaskModelViewMatrix = glGetUniformLocation(segmentationMaskInstancingShader, "ModelViewMatrix");
+	segmentationMaskProjectionMatrix = glGetUniformLocation(segmentationMaskInstancingShader, "ProjectionMatrix");
+
+	glUseProgram(0);
+
 	instancingShader = gltLoadShaderPair(instancingVertexShader, instancingFragmentShader);
 	glLinkProgram(instancingShader);
 	glUseProgram(instancingShader);
@@ -1413,7 +1464,6 @@ void GLInstancingRenderer::setProjectiveTextureMatrices(const float viewMatrix[1
 void GLInstancingRenderer::setProjectiveTexture(bool useProjectiveTexture)
 {
 	m_data->m_useProjectiveTexture = useProjectiveTexture;
-	useShadowMap = !useProjectiveTexture;
 }
 
 void GLInstancingRenderer::updateCamera(int upAxis)
@@ -1510,47 +1560,50 @@ void GLInstancingRenderer::renderScene()
 
 	//glFlush();
 
-	if (useShadowMap)
+	if (m_data->m_useProjectiveTexture)
-	{
-		renderSceneInternal(B3_CREATE_SHADOWMAP_RENDERMODE);
-
-		if (m_planeReflectionShapeIndex >= 0)
-		{
-			/* Don't update color or depth. */
-			glDisable(GL_DEPTH_TEST);
-			glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-
-			/* Draw 1 into the stencil buffer. */
-			glEnable(GL_STENCIL_TEST);
-			glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
-			glStencilFunc(GL_ALWAYS, 1, 0xffffffff);
-
-			/* Now render floor; floor pixels just get their stencil set to 1. */
-			renderSceneInternal(B3_USE_SHADOWMAP_RENDERMODE_REFLECTION_PLANE);
-
-			/* Re-enable update of color and depth. */
-			glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-			glEnable(GL_DEPTH_TEST);
-
-			/* Now, only render where stencil is set to 1. */
-			glStencilFunc(GL_EQUAL, 1, 0xffffffff); /* draw if ==1 */
-			glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-
-			//draw the reflection objects
-			renderSceneInternal(B3_USE_SHADOWMAP_RENDERMODE_REFLECTION);
-
-			glDisable(GL_STENCIL_TEST);
-		}
-
-		renderSceneInternal(B3_USE_SHADOWMAP_RENDERMODE);
-	}
-	else if (m_data->m_useProjectiveTexture)
 	{
 		renderSceneInternal(B3_USE_PROJECTIVE_TEXTURE_RENDERMODE);
 	}
 	else
 	{
-		renderSceneInternal();
+		if (useShadowMap)
+		{
+			renderSceneInternal(B3_CREATE_SHADOWMAP_RENDERMODE);
+
+			if (m_planeReflectionShapeIndex >= 0)
+			{
+				/* Don't update color or depth. */
+				glDisable(GL_DEPTH_TEST);
+				glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+
+				/* Draw 1 into the stencil buffer. */
+				glEnable(GL_STENCIL_TEST);
+				glStencilOp(GL_REPLACE, GL_REPLACE, GL_REPLACE);
+				glStencilFunc(GL_ALWAYS, 1, 0xffffffff);
+
+				/* Now render floor; floor pixels just get their stencil set to 1. */
+				renderSceneInternal(B3_USE_SHADOWMAP_RENDERMODE_REFLECTION_PLANE);
+
+				/* Re-enable update of color and depth. */
+				glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+				glEnable(GL_DEPTH_TEST);
+
+				/* Now, only render where stencil is set to 1. */
+				glStencilFunc(GL_EQUAL, 1, 0xffffffff); /* draw if ==1 */
+				glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+
+				//draw the reflection objects
+				renderSceneInternal(B3_USE_SHADOWMAP_RENDERMODE_REFLECTION);
+
+				glDisable(GL_STENCIL_TEST);
+			}
+
+			renderSceneInternal(B3_USE_SHADOWMAP_RENDERMODE);
+		}
+		else
+		{
+			renderSceneInternal();
+		}
 	}
 }
 
@@ -1970,7 +2023,7 @@ void GLInstancingRenderer::renderSceneInternal(int orgRenderMode)
 
 	if (!useShadowMap)
 	{
-		renderMode = B3_DEFAULT_RENDERMODE;
+		renderMode = orgRenderMode;
 	}
 
 	if (orgRenderMode == B3_USE_PROJECTIVE_TEXTURE_RENDERMODE)
@@ -2281,8 +2334,11 @@ void GLInstancingRenderer::renderSceneInternal(int orgRenderMode)
 				PointerCaster vertex;
 				vertex.m_baseIndex = gfxObj->m_vertexArrayOffset * vertexStride;
 
+				//vertex position
 				glVertexAttribPointer(0, 4, GL_FLOAT, GL_FALSE, 9 * sizeof(float), vertex.m_pointer);
+				//instance_position
 				glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, 0, (GLvoid*)(transparentInstances[i].m_instanceId * 4 * sizeof(float) + m_data->m_maxShapeCapacityInBytes));
+				//instance_quaternion
 				glVertexAttribPointer(2, 4, GL_FLOAT, GL_FALSE, 0, (GLvoid*)(transparentInstances[i].m_instanceId * 4 * sizeof(float) + m_data->m_maxShapeCapacityInBytes + POSITION_BUFFER_SIZE));
 
 				PointerCaster uv;
@@ -2293,8 +2349,10 @@ void GLInstancingRenderer::renderSceneInternal(int orgRenderMode)
 
 				glVertexAttribPointer(3, 2, GL_FLOAT, GL_FALSE, 9 * sizeof(float), uv.m_pointer);
 				glVertexAttribPointer(4, 3, GL_FLOAT, GL_FALSE, 9 * sizeof(float), normal.m_pointer);
+				//instance_color
 				glVertexAttribPointer(5, 4, GL_FLOAT, GL_FALSE, 0, (GLvoid*)(transparentInstances[i].m_instanceId * 4 * sizeof(float) + m_data->m_maxShapeCapacityInBytes + POSITION_BUFFER_SIZE + ORIENTATION_BUFFER_SIZE));
-				glVertexAttribPointer(6, 3, GL_FLOAT, GL_FALSE, 0, (GLvoid*)(transparentInstances[i].m_instanceId * 3 * sizeof(float) + m_data->m_maxShapeCapacityInBytes + POSITION_BUFFER_SIZE + ORIENTATION_BUFFER_SIZE + COLOR_BUFFER_SIZE));
+				//instance_scale
+				glVertexAttribPointer(6, 4, GL_FLOAT, GL_FALSE, 0, (GLvoid*)(transparentInstances[i].m_instanceId * 4 * sizeof(float) + m_data->m_maxShapeCapacityInBytes + POSITION_BUFFER_SIZE + ORIENTATION_BUFFER_SIZE + COLOR_BUFFER_SIZE));
 
 				glEnableVertexAttribArray(0);
 				glEnableVertexAttribArray(1);
@@ -2341,6 +2399,15 @@ void GLInstancingRenderer::renderSceneInternal(int orgRenderMode)
 					{
 						switch (renderMode)
 						{
+							case B3_SEGMENTATION_MASK_RENDERMODE:
+							{
+								glUseProgram(segmentationMaskInstancingShader);
+								glUniformMatrix4fv(segmentationMaskProjectionMatrix, 1, false, &m_data->m_projectionMatrix[0]);
+								glUniformMatrix4fv(segmentationMaskModelViewMatrix, 1, false, &m_data->m_viewMatrix[0]);
+								glDrawElementsInstanced(GL_TRIANGLES, indexCount, GL_UNSIGNED_INT, indexOffset, gfxObj->m_numGraphicsInstances);
+
+								break;
+							}
 							case B3_DEFAULT_RENDERMODE:
 							{
 								if (gfxObj->m_flags & eGfxTransparency)

examples/OpenGLWindow/Shaders/createShadowMapInstancingVS.glsl

@@ -8,7 +8,7 @@ layout (location = 2) in vec4 instance_quaternion;
 layout (location = 3) in vec2 uvcoords;
 layout (location = 4) in vec3 vertexnormal;
 layout (location = 5) in vec4 instance_color;
-layout (location = 6) in vec3 instance_scale;
+layout (location = 6) in vec4 instance_scale_obUid;
 
 
 uniform mat4 depthMVP;
@@ -48,7 +48,7 @@ vec4 quatRotate ( in vec4 p, in vec4 q )
 void main(void)
 {
 	vec4 q = instance_quaternion;
-	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);
+	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);
 	vec4 vertexPos = depthMVP * vec4( (instance_position+localcoord).xyz,1);
 	gl_Position = vertexPos;
 }

examples/OpenGLWindow/Shaders/createShadowMapInstancingVS.h

@@ -8,7 +8,7 @@ static const char* createShadowMapInstancingVertexShader =
 	"layout (location = 3) in vec2 uvcoords;\n"
 	"layout (location = 4) in vec3 vertexnormal;\n"
 	"layout (location = 5) in vec4 instance_color;\n"
-	"layout (location = 6) in vec3 instance_scale;\n"
+	"layout (location = 6) in vec4 instance_scale_obUid;\n"
 	"uniform mat4 depthMVP;\n"
 	"vec4 quatMul ( in vec4 q1, in vec4 q2 )\n"
 	"{\n"
@@ -41,7 +41,7 @@ static const char* createShadowMapInstancingVertexShader =
 	"void main(void)\n"
 	"{\n"
 	"	vec4 q = instance_quaternion;\n"
-	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);\n"
+	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);\n"
 	"	vec4 vertexPos = depthMVP * vec4( (instance_position+localcoord).xyz,1);\n"
 	"	gl_Position = vertexPos;\n"
 	"}\n";

examples/OpenGLWindow/Shaders/instancingVS.glsl

@@ -8,7 +8,7 @@ layout (location = 2) in vec4 instance_quaternion;
 layout (location = 3) in vec2 uvcoords;
 layout (location = 4) in vec3 vertexnormal;
 layout (location = 5) in vec4 instance_color;
-layout (location = 6) in vec3 instance_scale;
+layout (location = 6) in vec4 instance_scale_obUid;
 
 
 uniform mat4 ModelViewMatrix;
@@ -68,7 +68,7 @@ void main(void)
 	
 	lightDir = lightDirIn;
 	
-	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);
+	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);
 	vec4 vertexPos = ProjectionMatrix * ModelViewMatrix *(instance_position+localcoord);
 
 	gl_Position = vertexPos;

examples/OpenGLWindow/Shaders/instancingVS.h

@@ -8,7 +8,7 @@ static const char* instancingVertexShader =
 	"layout (location = 3) in vec2 uvcoords;\n"
 	"layout (location = 4) in vec3 vertexnormal;\n"
 	"layout (location = 5) in vec4 instance_color;\n"
-	"layout (location = 6) in vec3 instance_scale;\n"
+	"layout (location = 6) in vec4 instance_scale_obUid;\n"
 	"uniform mat4 ModelViewMatrix;\n"
 	"uniform mat4 ProjectionMatrix;\n"
 	"uniform vec3 lightDirIn;\n"
@@ -59,7 +59,7 @@ static const char* instancingVertexShader =
 	"	\n"
 	"	lightDir = lightDirIn;\n"
 	"	\n"
-	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);\n"
+	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);\n"
 	"	vec4 vertexPos = ProjectionMatrix * ModelViewMatrix *(instance_position+localcoord);\n"
 	"	gl_Position = vertexPos;\n"
 	"	\n"

examples/OpenGLWindow/Shaders/pointSpriteVS.glsl

@@ -8,7 +8,7 @@ layout (location = 1) in vec4 instance_position;
 layout (location = 3) in vec2 uvcoords;
 layout (location = 4) in vec3 vertexnormal;
 layout (location = 5) in vec4 instance_color;
-layout (location = 6) in vec3 instance_scale;
+layout (location = 6) in vec4 instance_scale_obUid;
 
 
 uniform float screenWidth = 700.f;
@@ -38,7 +38,7 @@ void main(void)
 	vec3 posEye = vec3(ModelViewMatrix * vec4(instance_position.xyz, 1.0));
    float dist = length(posEye);
 	float pointRadius = 1.f;
-    gl_PointSize = instance_scale.x * pointRadius * (screenWidth / dist);
+    gl_PointSize = instance_scale_obUid.x * pointRadius * (screenWidth / dist);
 
 	gl_Position = vertexPos;
 	

examples/OpenGLWindow/Shaders/pointSpriteVS.h

@@ -7,7 +7,7 @@ static const char* pointSpriteVertexShader =
 	"layout (location = 3) in vec2 uvcoords;\n"
 	"layout (location = 4) in vec3 vertexnormal;\n"
 	"layout (location = 5) in vec4 instance_color;\n"
-	"layout (location = 6) in vec3 instance_scale;\n"
+	"layout (location = 6) in vec4 instance_scale_obUid;\n"
 	"uniform float screenWidth = 700.f;\n"
 	"uniform mat4 ModelViewMatrix;\n"
 	"uniform mat4 ProjectionMatrix;\n"
@@ -29,7 +29,7 @@ static const char* pointSpriteVertexShader =
 	"	vec3 posEye = vec3(ModelViewMatrix * vec4(instance_position.xyz, 1.0));\n"
 	"   float dist = length(posEye);\n"
 	"	float pointRadius = 1.f;\n"
-	"    gl_PointSize = instance_scale.x * pointRadius * (screenWidth / dist);\n"
+	"    gl_PointSize = instance_scale_obUid.x * pointRadius * (screenWidth / dist);\n"
 	"	gl_Position = vertexPos;\n"
 	"	\n"
 	"	fragment.color = instance_color;\n"

examples/OpenGLWindow/Shaders/projectiveTextureInstancingVS.glsl

@@ -8,7 +8,7 @@ layout (location = 2) in vec4 instance_quaternion;
 layout (location = 3) in vec2 uvcoords;
 layout (location = 4) in vec3 vertexnormal;
 layout (location = 5) in vec4 instance_color;
-layout (location = 6) in vec3 instance_scale;
+layout (location = 6) in vec4 instance_scale_obUid;
 
 
 uniform mat4 TextureMVP;
@@ -86,7 +86,7 @@ void main(void)
 	lightSpecularIntensity = lightSpecularIntensityIn;
 	materialSpecularColor = materialSpecularColorIn;
 	
-	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);
+	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);
 	vertexPos = vec4((instance_position+localcoord).xyz,1);
 	
 	vec4 vertexLoc = MVP* vec4((instance_position+localcoord).xyz,1);

examples/OpenGLWindow/Shaders/projectiveTextureInstancingVS.h

@@ -8,7 +8,7 @@ static const char* projectiveTextureInstancingVertexShader =
 	"layout (location = 3) in vec2 uvcoords;\n"
 	"layout (location = 4) in vec3 vertexnormal;\n"
 	"layout (location = 5) in vec4 instance_color;\n"
-	"layout (location = 6) in vec3 instance_scale;\n"
+	"layout (location = 6) in vec4 instance_scale_obUid;\n"
 	"uniform mat4 TextureMVP;\n"
 	"uniform mat4 MVP;\n"
 	"uniform vec3 lightPosIn;\n"
@@ -74,7 +74,7 @@ static const char* projectiveTextureInstancingVertexShader =
 	"	lightSpecularIntensity = lightSpecularIntensityIn;\n"
 	"	materialSpecularColor = materialSpecularColorIn;\n"
 	"	\n"
-	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);\n"
+	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);\n"
 	"	vertexPos = vec4((instance_position+localcoord).xyz,1);\n"
 	"	\n"
 	"	vec4 vertexLoc = MVP* vec4((instance_position+localcoord).xyz,1);\n"

examples/OpenGLWindow/Shaders/segmentationMaskInstancingPS.glsl

@@ -0,0 +1,16 @@
+#version 330
+precision highp float;
+in vec4 scale_obuid;
+out vec4 color;
+
+
+
+void main(void)
+{
+	highp int obuid = int(scale_obuid.w);
+	float r = 1*1.f/255.f;//((obuid>>0 )&0xff)*(1./255.f);
+	float g = ((obuid>>8 )&0xff)*(1./255.f);
+	float b = ((obuid>>16)&0xff)*(1./255.f);
+	float a = ((obuid>>24)&0xff)*(1./255.f);
+	color  = vec4(r,g,b,a);
+}

examples/OpenGLWindow/Shaders/segmentationMaskInstancingPS.h

@@ -0,0 +1,15 @@
+//this file is autogenerated using stringify.bat (premake --stringify) in the build folder of this project
+static const char* segmentationMaskInstancingFragmentShader =
+	"#version 330\n"
+	"precision highp float;\n"
+	"in vec4 scale_obuid;\n"
+	"out vec4 color;\n"
+	"void main(void)\n"
+	"{\n"
+	"	highp int obuid = int(scale_obuid.w);\n"
+	"	float r = ((obuid>>0 )&0xff)*(1./255.f);\n"
+	"	float g = ((obuid>>8 )&0xff)*(1./255.f);\n"
+	"	float b = ((obuid>>16)&0xff)*(1./255.f);\n"
+	"	float a = ((obuid>>24)&0xff)*(1./255.f);\n"
+	"	color  = vec4(r,g,b,a);\n"
+	"}\n";

examples/OpenGLWindow/Shaders/segmentationMaskInstancingVS.glsl

@@ -0,0 +1,61 @@
+#version 330
+precision highp float;
+
+
+layout (location = 0) in vec4 position;
+layout (location = 1) in vec4 instance_position;
+layout (location = 2) in vec4 instance_quaternion;
+layout (location = 5) in vec4 instance_color;
+layout (location = 6) in vec4 instance_scale_obUid;
+
+uniform mat4 ModelViewMatrix;
+uniform mat4 ProjectionMatrix;
+
+out vec4 scale_obuid;
+
+out Fragment
+{
+     vec4 color;
+} fragment;
+
+vec4 quatMul ( in vec4 q1, in vec4 q2 )
+{
+    vec3  im = q1.w * q2.xyz + q1.xyz * q2.w + cross ( q1.xyz, q2.xyz );
+    vec4  dt = q1 * q2;
+    float re = dot ( dt, vec4 ( -1.0, -1.0, -1.0, 1.0 ) );
+    return vec4 ( im, re );
+}
+
+vec4 quatFromAxisAngle(vec4 axis, in float angle)
+{
+    float cah = cos(angle*0.5);
+    float sah = sin(angle*0.5);
+	float d = inversesqrt(dot(axis,axis));
+	vec4 q = vec4(axis.x*sah*d,axis.y*sah*d,axis.z*sah*d,cah);
+	return q;
+}
+//
+// vector rotation via quaternion
+//
+vec4 quatRotate3 ( in vec3 p, in vec4 q )
+{
+    vec4 temp = quatMul ( q, vec4 ( p, 0.0 ) );
+    return quatMul ( temp, vec4 ( -q.x, -q.y, -q.z, q.w ) );
+}
+vec4 quatRotate ( in vec4 p, in vec4 q )
+{
+    vec4 temp = quatMul ( q, p );
+    return quatMul ( temp, vec4 ( -q.x, -q.y, -q.z, q.w ) );
+}
+
+
+void main(void)
+{
+	vec4 q = instance_quaternion;
+	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);
+	vec4 vertexPos = ProjectionMatrix * ModelViewMatrix *(instance_position+localcoord);
+	scale_obuid = instance_scale_obUid;
+	gl_Position = vertexPos;
+	fragment.color = instance_color;
+}
+

examples/OpenGLWindow/Shaders/segmentationMaskInstancingVS.h

@@ -0,0 +1,53 @@
+//this file is autogenerated using stringify.bat (premake --stringify) in the build folder of this project
+static const char* segmentationMaskInstancingVertexShader =
+	"#version 330\n"
+	"precision highp float;\n"
+	"layout (location = 0) in vec4 position;\n"
+	"layout (location = 1) in vec4 instance_position;\n"
+	"layout (location = 2) in vec4 instance_quaternion;\n"
+	"layout (location = 5) in vec4 instance_color;\n"
+	"layout (location = 6) in vec4 instance_scale_obUid;\n"
+	"uniform mat4 ModelViewMatrix;\n"
+	"uniform mat4 ProjectionMatrix;\n"
+	"out vec4 scale_obuid;\n"
+	"out Fragment\n"
+	"{\n"
+	"     vec4 color;\n"
+	"} fragment;\n"
+	"vec4 quatMul ( in vec4 q1, in vec4 q2 )\n"
+	"{\n"
+	"    vec3  im = q1.w * q2.xyz + q1.xyz * q2.w + cross ( q1.xyz, q2.xyz );\n"
+	"    vec4  dt = q1 * q2;\n"
+	"    float re = dot ( dt, vec4 ( -1.0, -1.0, -1.0, 1.0 ) );\n"
+	"    return vec4 ( im, re );\n"
+	"}\n"
+	"vec4 quatFromAxisAngle(vec4 axis, in float angle)\n"
+	"{\n"
+	"    float cah = cos(angle*0.5);\n"
+	"    float sah = sin(angle*0.5);\n"
+	"	float d = inversesqrt(dot(axis,axis));\n"
+	"	vec4 q = vec4(axis.x*sah*d,axis.y*sah*d,axis.z*sah*d,cah);\n"
+	"	return q;\n"
+	"}\n"
+	"//\n"
+	"// vector rotation via quaternion\n"
+	"//\n"
+	"vec4 quatRotate3 ( in vec3 p, in vec4 q )\n"
+	"{\n"
+	"    vec4 temp = quatMul ( q, vec4 ( p, 0.0 ) );\n"
+	"    return quatMul ( temp, vec4 ( -q.x, -q.y, -q.z, q.w ) );\n"
+	"}\n"
+	"vec4 quatRotate ( in vec4 p, in vec4 q )\n"
+	"{\n"
+	"    vec4 temp = quatMul ( q, p );\n"
+	"    return quatMul ( temp, vec4 ( -q.x, -q.y, -q.z, q.w ) );\n"
+	"}\n"
+	"void main(void)\n"
+	"{\n"
+	"	vec4 q = instance_quaternion;\n"
+	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);\n"
+	"	vec4 vertexPos = ProjectionMatrix * ModelViewMatrix *(instance_position+localcoord);\n"
+	"	scale_obuid = instance_scale_obUid;\n"
+	"	gl_Position = vertexPos;\n"
+	"	fragment.color = instance_color;\n"
+	"}\n";

examples/OpenGLWindow/Shaders/useShadowMapInstancingVS.glsl

@@ -8,7 +8,7 @@ layout (location = 2) in vec4 instance_quaternion;
 layout (location = 3) in vec2 uvcoords;
 layout (location = 4) in vec3 vertexnormal;
 layout (location = 5) in vec4 instance_color;
-layout (location = 6) in vec3 instance_scale;
+layout (location = 6) in vec4 instance_scale_obUid;
 
 
 uniform mat4 ModelViewMatrix;
@@ -88,7 +88,7 @@ void main(void)
 	lightSpecularIntensity = lightSpecularIntensityIn;
 	materialSpecularColor = materialSpecularColorIn;
 	
-	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);
+	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);
 	vertexPos = vec4((instance_position+localcoord).xyz,1);
 	
 	vec4 vertexLoc = MVP* vec4((instance_position+localcoord).xyz,1);

examples/OpenGLWindow/Shaders/useShadowMapInstancingVS.h

@@ -8,7 +8,7 @@ static const char* useShadowMapInstancingVertexShader =
 	"layout (location = 3) in vec2 uvcoords;\n"
 	"layout (location = 4) in vec3 vertexnormal;\n"
 	"layout (location = 5) in vec4 instance_color;\n"
-	"layout (location = 6) in vec3 instance_scale;\n"
+	"layout (location = 6) in vec4 instance_scale_obUid;\n"
 	"uniform mat4 ModelViewMatrix;\n"
 	"uniform mat4 ProjectionMatrix;\n"
 	"uniform mat4 DepthBiasModelViewProjectionMatrix;\n"
@@ -76,7 +76,7 @@ static const char* useShadowMapInstancingVertexShader =
 	"	lightSpecularIntensity = lightSpecularIntensityIn;\n"
 	"	materialSpecularColor = materialSpecularColorIn;\n"
 	"	\n"
-	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale,q);\n"
+	"	vec4 localcoord = quatRotate3( position.xyz*instance_scale_obUid.xyz,q);\n"
 	"	vertexPos = vec4((instance_position+localcoord).xyz,1);\n"
 	"	\n"
 	"	vec4 vertexLoc = MVP* vec4((instance_position+localcoord).xyz,1);\n"

examples/OpenGLWindow/SimpleOpenGL3App.cpp

@@ -63,6 +63,26 @@ struct SimpleInternalData
 	GLRenderToTexture* m_renderTexture;
 	void* m_userPointer;
 	int m_upAxis;  //y=1 or z=2 is supported
+	int m_customViewPortWidth;
+	int m_customViewPortHeight;
+	SimpleInternalData()
+		:m_fontTextureId(0),
+		m_largeFontTextureId(0),
+		m_fontStash(0),
+		m_fontStash2(0),
+		m_renderCallbacks(0),
+		m_renderCallbacks2(0),
+		m_droidRegular(0),
+		m_droidRegular2(0),
+		m_frameDumpPngFileName(0),
+		m_ffmpegFile(0),
+		m_renderTexture(0),
+		m_userPointer(0),
+		m_upAxis(0),
+		m_customViewPortWidth(-1),
+		m_customViewPortHeight(-1)
+		{
+		}
 };
 
 static SimpleOpenGL3App* gApp = 0;
@@ -288,11 +308,7 @@ SimpleOpenGL3App::SimpleOpenGL3App(const char* title, int width, int height, boo
 	gApp = this;
 
 	m_data = new SimpleInternalData;
-	m_data->m_frameDumpPngFileName = 0;
+	
-	m_data->m_renderTexture = 0;
-	m_data->m_ffmpegFile = 0;
-	m_data->m_userPointer = 0;
-	m_data->m_upAxis = 1;
 
 	if (windowType == 0)
 	{
@@ -909,10 +925,26 @@ SimpleOpenGL3App::~SimpleOpenGL3App()
 	delete m_data;
 }
 
+
+void SimpleOpenGL3App::setViewport(int width, int height)
+{
+	m_data->m_customViewPortWidth=width;
+	m_data->m_customViewPortHeight=height;
+	if (width>=0)
+	{
+		glViewport(0,0,width,height);
+	} else
+	{
+		glViewport(0,0,m_window->getRetinaScale()*m_instancingRenderer->getScreenWidth(),m_window->getRetinaScale()*m_instancingRenderer->getScreenHeight());
+	}
+}
+
 void SimpleOpenGL3App::getScreenPixels(unsigned char* rgbaBuffer, int bufferSizeInBytes, float* depthBuffer, int depthBufferSizeInBytes)
 {
-	int width = (int)m_window->getRetinaScale() * m_instancingRenderer->getScreenWidth();
+	int width = m_data->m_customViewPortWidth>=0? m_data->m_customViewPortWidth : (int)m_window->getRetinaScale() * m_instancingRenderer->getScreenWidth();
-	int height = (int)m_window->getRetinaScale() * m_instancingRenderer->getScreenHeight();
+	int height = m_data->m_customViewPortHeight >=0? m_data->m_customViewPortHeight: (int)m_window->getRetinaScale() * m_instancingRenderer->getScreenHeight();
+
+	b3Assert((width * height * 4) == bufferSizeInBytes);
 	if ((width * height * 4) == bufferSizeInBytes)
 	{
 		glReadPixels(0, 0, width, height, GL_RGBA, GL_UNSIGNED_BYTE, rgbaBuffer);
@@ -920,6 +952,7 @@ void SimpleOpenGL3App::getScreenPixels(unsigned char* rgbaBuffer, int bufferSize
 		glstat = glGetError();
 		b3Assert(glstat == GL_NO_ERROR);
 	}
+	b3Assert((width * height * sizeof(float)) == depthBufferSizeInBytes);
 	if ((width * height * sizeof(float)) == depthBufferSizeInBytes)
 	{
 		glReadPixels(0, 0, width, height, GL_DEPTH_COMPONENT, GL_FLOAT, depthBuffer);

examples/OpenGLWindow/SimpleOpenGL3App.h

@@ -24,7 +24,8 @@ struct SimpleOpenGL3App : public CommonGraphicsApp
 	virtual void registerGrid(int xres, int yres, float color0[4], float color1[4]);
 	void dumpNextFrameToPng(const char* pngFilename);
 	void dumpFramesToVideo(const char* mp4Filename);
-	void getScreenPixels(unsigned char* rgbaBuffer, int bufferSizeInBytes, float* depthBuffer, int depthBufferSizeInBytes);
+	virtual void getScreenPixels(unsigned char* rgbaBuffer, int bufferSizeInBytes, float* depthBuffer, int depthBufferSizeInBytes);
+	virtual void setViewport(int width, int height);
 
 	void drawGrid(DrawGridData data = DrawGridData());
 	virtual void setUpAxis(int axis);

examples/SharedMemory/PhysicsClientC_API.cpp

@@ -4849,7 +4849,7 @@ B3_SHARED_API b3SharedMemoryCommandHandle b3InitConfigureOpenGLVisualizer(b3Phys
 {
 	PhysicsClient* cl = (PhysicsClient*)physClient;
 	b3Assert(cl);
-	b3Assert(cl->canSubmitCommand());
+	//b3Assert(cl->canSubmitCommand());
 	struct SharedMemoryCommand* command = cl->getAvailableSharedMemoryCommand();
 	b3Assert(command);
 

examples/SharedMemory/PhysicsServerCommandProcessor.cpp

@@ -1631,6 +1631,7 @@ struct PhysicsServerCommandProcessorInternalData
 	MyBroadphaseCallback m_cachedOverlappingObjects;
 
 	btAlignedObjectArray<int> m_sdfRecentLoadedBodies;
+	btAlignedObjectArray<int> m_graphicsIndexToSegmentationMask;
 
 	btAlignedObjectArray<InternalStateLogger*> m_stateLoggers;
 	int m_stateLoggersUniqueId;
@@ -1725,6 +1726,7 @@ struct PhysicsServerCommandProcessorInternalData
 			}
 #endif  //ENABLE_STATIC_GRPC_PLUGIN
 
+
 #ifdef STATIC_EGLRENDERER_PLUGIN
 			{
 				bool initPlugin = false;
@@ -1733,12 +1735,15 @@ struct PhysicsServerCommandProcessorInternalData
 			}
 #endif  //STATIC_EGLRENDERER_PLUGIN
 
+
 #ifndef SKIP_STATIC_TINYRENDERER_PLUGIN
 			{
 				int renderPluginId = m_pluginManager.registerStaticLinkedPlugin("tinyRendererPlugin", initPlugin_tinyRendererPlugin, exitPlugin_tinyRendererPlugin, executePluginCommand_tinyRendererPlugin, 0, 0, getRenderInterface_tinyRendererPlugin, 0, 0);
 				m_pluginManager.selectPluginRenderer(renderPluginId);
 			}
 #endif
+
+
 		}
 
 		m_vrControllerEvents.init();
@@ -2607,6 +2612,7 @@ void PhysicsServerCommandProcessor::deleteDynamicsWorld()
 	m_data->m_meshInterfaces.clear();
 	m_data->m_collisionShapes.clear();
 	m_data->m_bulletCollisionShape2UrdfCollision.clear();
+	m_data->m_graphicsIndexToSegmentationMask.clear();
 
 	delete m_data->m_dynamicsWorld;
 	m_data->m_dynamicsWorld = 0;
@@ -2737,12 +2743,33 @@ bool PhysicsServerCommandProcessor::processImportedObjects(const char* fileName,
 		if (mb)
 			mb->setUserIndex2(bodyUniqueId);
 
+
+
 		if (mb)
 		{
 			bodyHandle->m_multiBody = mb;
 
 			m_data->m_sdfRecentLoadedBodies.push_back(bodyUniqueId);
 
+			 
+			int segmentationMask = bodyUniqueId;
+			
+			{
+				int graphicsIndex = -1;
+				if (mb->getBaseCollider())
+				{
+					graphicsIndex = mb->getBaseCollider()->getUserIndex();
+				}
+				if (graphicsIndex>=0)
+				{
+					if (m_data->m_graphicsIndexToSegmentationMask.size()<(graphicsIndex+1))
+					{
+						m_data->m_graphicsIndexToSegmentationMask.resize(graphicsIndex+1);
+					}
+					m_data->m_graphicsIndexToSegmentationMask[graphicsIndex]= segmentationMask;
+				}
+			}
+
 			createJointMotors(mb);
 
 #ifdef B3_ENABLE_TINY_AUDIO
@@ -2780,6 +2807,24 @@ bool PhysicsServerCommandProcessor::processImportedObjects(const char* fileName,
 
 				mb->getLink(i).m_linkName = linkName->c_str();
 
+				{
+					int graphicsIndex = -1;
+					if (mb->getLinkCollider(i))
+					{
+							graphicsIndex = mb->getLinkCollider(i)->getUserIndex();
+					}
+					if (graphicsIndex>=0)
+					{
+						int linkIndex = i;
+						if (m_data->m_graphicsIndexToSegmentationMask.size()<(graphicsIndex+1))
+						{
+							m_data->m_graphicsIndexToSegmentationMask.resize(graphicsIndex+1);
+						}
+						int segmentationMask = bodyUniqueId + ((linkIndex + 1) << 24);
+						m_data->m_graphicsIndexToSegmentationMask[graphicsIndex]= segmentationMask;
+					}
+				}
+				
 				std::string* jointName = new std::string(u2b.getJointName(urdfLinkIndex).c_str());
 				m_data->m_strings.push_back(jointName);
 
@@ -2808,6 +2853,23 @@ bool PhysicsServerCommandProcessor::processImportedObjects(const char* fileName,
 		}
 		else
 		{
+			int segmentationMask = bodyUniqueId;
+			if (rb)
+			{
+				int graphicsIndex = -1;
+				{
+					graphicsIndex = rb->getUserIndex();
+				}
+				if (graphicsIndex>=0)
+				{
+					if (m_data->m_graphicsIndexToSegmentationMask.size()<(graphicsIndex+1))
+					{
+						m_data->m_graphicsIndexToSegmentationMask.resize(graphicsIndex+1);
+					}
+					m_data->m_graphicsIndexToSegmentationMask[graphicsIndex]= segmentationMask;
+				}
+			}
+
 			//b3Warning("No multibody loaded from URDF. Could add btRigidBody+btTypedConstraint solution later.");
 			bodyHandle->m_rigidBody = rb;
 			rb->setUserIndex2(bodyUniqueId);
@@ -3462,6 +3524,10 @@ bool PhysicsServerCommandProcessor::processRequestCameraImageCommand(const struc
 				this->m_data->m_guiHelper->setProjectiveTexture(false);
 			}
 
+			if ((flags & (ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX + ER_SEGMENTATION_MASK))==0)
+			{
+				segmentationMaskBuffer=0;
+			}
 			m_data->m_guiHelper->copyCameraImageData(viewMat,
 													 projMat, pixelRGBA, numRequestedPixels,
 													 depthBuffer, numRequestedPixels,
@@ -3470,11 +3536,36 @@ bool PhysicsServerCommandProcessor::processRequestCameraImageCommand(const struc
 
 			if (numPixelsCopied > 0)
 			{
+				//convert segmentation mask
+
+				if (segmentationMaskBuffer)
+				{
+					for (int i=0;i<numPixelsCopied;i++)
+					{
+						int graphicsSegMask = segmentationMaskBuffer[i];
+						int segMask = -1;
+						if ((graphicsSegMask >= 0) && (graphicsSegMask<m_data->m_graphicsIndexToSegmentationMask.size()))
+						{
+							segMask = m_data->m_graphicsIndexToSegmentationMask[graphicsSegMask];
+						}
+						if ((flags & ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX) == 0)
+						{
+							if (segMask >= 0)
+							{
+								segMask &= ((1 << 24) - 1);
+							}
+						}
+						segmentationMaskBuffer[i] = segMask;
+					}
+					
+				}
+
+
 				handled = true;
 				m_data->m_guiHelper->debugDisplayCameraImageData(viewMat,
 																 projMat, pixelRGBA, numRequestedPixels,
 																 depthBuffer, numRequestedPixels,
-																 0, numRequestedPixels,
+																 segmentationMaskBuffer, numRequestedPixels,
 																 startPixelIndex, width, height, &numPixelsCopied);
 			}
 		}
@@ -8459,6 +8550,7 @@ bool PhysicsServerCommandProcessor::processRemoveBodyCommand(const struct Shared
 			m_data->m_bodyHandles.freeHandle(bodyUniqueId);
 		}
 	}
+
 	for (int i = 0; i < clientCmd.m_removeObjectArgs.m_numUserCollisionShapes; i++)
 	{
 		int removeCollisionShapeId = clientCmd.m_removeObjectArgs.m_userCollisionShapes[i];
@@ -8492,7 +8584,7 @@ bool PhysicsServerCommandProcessor::processRemoveBodyCommand(const struct Shared
 				}
 				if (foundIndex >= 0)
 				{
-					btMultiBodyWorldImporter* importer = m_data->m_worldImporters[i];
+					btMultiBodyWorldImporter* importer = m_data->m_worldImporters[foundIndex];
 					m_data->m_worldImporters.removeAtIndex(foundIndex);
 					delete importer;
 					m_data->m_userCollisionShapeHandles.freeHandle(removeCollisionShapeId);

examples/SharedMemory/SharedMemoryPublic.h

@@ -746,6 +746,7 @@ enum EnumRendererAuxFlags
 {
 	ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX = 1,
 	ER_USE_PROJECTIVE_TEXTURE = 2,
+	ER_SEGMENTATION_MASK=4,
 };
 
 ///flags to pick the IK solver and other options

examples/SharedMemory/plugins/eglPlugin/eglRendererVisualShapeConverter.cpp

@@ -31,6 +31,7 @@ subject to the following restrictions:
 #include "../SharedMemory/SharedMemoryPublic.h"  //for b3VisualShapeData
 #include "../TinyRenderer/model.h"
 #include "stb_image/stb_image.h"
+#include "LinearMath/btMinMax.h"
 
 #ifdef __APPLE__
 #include "OpenGLWindow/MacOpenGLWindow.h"
@@ -96,6 +97,11 @@ struct EGLRendererVisualShapeConverterInternalData
 	btAlignedObjectArray<unsigned char> m_rgbaPixelBuffer1;
 	btAlignedObjectArray<float> m_depthBuffer1;
 
+	btAlignedObjectArray<unsigned char> m_segmentationMaskSourceRgbaPixelBuffer;
+	btAlignedObjectArray<float> m_segmentationMaskSourceDepthBuffer;
+
+	
+	btAlignedObjectArray<int> m_graphicsIndexToSegmentationMask;
 	btHashMap<btHashInt, EGLRendererObjectArray*> m_swRenderInstances;
 
 	btAlignedObjectArray<b3VisualShapeData> m_visualShapes;
@@ -103,6 +109,9 @@ struct EGLRendererVisualShapeConverterInternalData
 	int m_upAxis;
 	int m_swWidth;
 	int m_swHeight;
+	btAlignedObjectArray<unsigned char> m_sourceRgbaPixelBuffer;
+	btAlignedObjectArray<float> m_sourceDepthBuffer;
+
 	TGAImage m_rgbColorBuffer;
 	b3AlignedObjectArray<MyTexture3> m_textures;
 	b3AlignedObjectArray<float> m_depthBuffer;
@@ -783,6 +792,21 @@ void EGLRendererVisualShapeConverter::convertVisualShapes(
 					double scaling[3] = {1, 1, 1};
 					visuals->m_graphicsInstanceId = m_data->m_instancingRenderer->registerGraphicsInstance(shapeIndex, &visualShape.m_localVisualFrame[0], &visualShape.m_localVisualFrame[3], &visualShape.m_rgbaColor[0], scaling);
 
+					int segmentationMask = bodyUniqueId + ((linkIndex + 1) << 24);
+					{
+						int graphicsIndex = visuals->m_graphicsInstanceId;
+						if (graphicsIndex>=0)
+						{
+							if (m_data->m_graphicsIndexToSegmentationMask.size()<(graphicsIndex+1))
+							{
+								m_data->m_graphicsIndexToSegmentationMask.resize(graphicsIndex+1);
+							}
+							m_data->m_graphicsIndexToSegmentationMask[graphicsIndex]= segmentationMask;
+						}
+					}
+
+
+
 					m_data->m_instancingRenderer->writeTransforms();
 				}
 			}
@@ -923,29 +947,7 @@ void EGLRendererVisualShapeConverter::clearBuffers(TGAColor& clearColor)
 
 void EGLRendererVisualShapeConverter::render()
 {
-	m_data->m_window->endRendering();
+	//mode the the actual render code inside 'copyImageData' since we need to know the width/height
-	m_data->m_window->startRendering();
-	/*
-    ATTRIBUTE_ALIGNED16(float viewMat[16]);
-    ATTRIBUTE_ALIGNED16(float projMat[16]);
-    m_data->m_camera.getCameraProjectionMatrix(projMat);
-    m_data->m_camera.getCameraViewMatrix(viewMat);
-    cout<<viewMat[4*0 + 0]<<" "<<viewMat[4*0+1]<<" "<<viewMat[4*0+2]<<" "<<viewMat[4*0+3] << endl;
-    cout<<viewMat[4*1 + 0]<<" "<<viewMat[4*1+1]<<" "<<viewMat[4*1+2]<<" "<<viewMat[4*1+3] << endl;
-    cout<<viewMat[4*2 + 0]<<" "<<viewMat[4*2+1]<<" "<<viewMat[4*2+2]<<" "<<viewMat[4*2+3] << endl;
-    cout<<viewMat[4*3 + 0]<<" "<<viewMat[4*3+1]<<" "<<viewMat[4*3+2]<<" "<<viewMat[4*3+3] << endl;
-    */
-
-	B3_PROFILE("m_instancingRenderer render");
-	m_data->m_instancingRenderer->writeTransforms();
-	if (m_data->m_hasLightDirection)
-	{
-		m_data->m_instancingRenderer->setLightPosition(m_data->m_lightDirection);
-	}
-	m_data->m_instancingRenderer->setActiveCamera(&m_data->m_camera);
-	m_data->m_instancingRenderer->updateCamera(m_data->m_upAxis);
-
-	m_data->m_instancingRenderer->renderScene();
 }
 
 void EGLRendererVisualShapeConverter::render(const float viewMat[16], const float projMat[16])
@@ -991,90 +993,196 @@ void EGLRendererVisualShapeConverter::copyCameraImageDataGL(
 	int* segmentationMaskBuffer, int segmentationMaskSizeInPixels,
 	int startPixelIndex, int* widthPtr, int* heightPtr, int* numPixelsCopied)
 {
-	int sourceWidth = m_data->m_window->getWidth() * m_data->m_window->getRetinaScale();
-	int sourceHeight = m_data->m_window->getHeight() * m_data->m_window->getRetinaScale();
-
 	if (numPixelsCopied)
 		*numPixelsCopied = 0;
 
+	int destinationWidth = *widthPtr;
+	int destinationHeight = *heightPtr;
+	int sourceWidth = btMin(destinationWidth, (int)( m_data->m_window->getWidth() * m_data->m_window->getRetinaScale()));
+	int sourceHeight = btMin(destinationHeight, (int)(m_data->m_window->getHeight() * m_data->m_window->getRetinaScale()));
+
+	
+
 	int numTotalPixels = (*widthPtr) * (*heightPtr);
 	int numRemainingPixels = numTotalPixels - startPixelIndex;
 	int numBytesPerPixel = 4;  //RGBA
 	int numRequestedPixels = btMin(rgbaBufferSizeInPixels, numRemainingPixels);
 	if (numRequestedPixels)
 	{
-		if (startPixelIndex == 0)
+		
+		if (startPixelIndex==0)
 		{
+			glViewport(0,0,sourceWidth,sourceHeight);
+			m_data->m_window->endRendering();
+			m_data->m_window->startRendering();
+			/*
+			ATTRIBUTE_ALIGNED16(float viewMat[16]);
+			ATTRIBUTE_ALIGNED16(float projMat[16]);
+			m_data->m_camera.getCameraProjectionMatrix(projMat);
+			m_data->m_camera.getCameraViewMatrix(viewMat);
+			cout<<viewMat[4*0 + 0]<<" "<<viewMat[4*0+1]<<" "<<viewMat[4*0+2]<<" "<<viewMat[4*0+3] << endl;
+			cout<<viewMat[4*1 + 0]<<" "<<viewMat[4*1+1]<<" "<<viewMat[4*1+2]<<" "<<viewMat[4*1+3] << endl;
+			cout<<viewMat[4*2 + 0]<<" "<<viewMat[4*2+1]<<" "<<viewMat[4*2+2]<<" "<<viewMat[4*2+3] << endl;
+			cout<<viewMat[4*3 + 0]<<" "<<viewMat[4*3+1]<<" "<<viewMat[4*3+2]<<" "<<viewMat[4*3+3] << endl;
+			*/
+
+			B3_PROFILE("m_instancingRenderer render");
+			m_data->m_instancingRenderer->writeTransforms();
+			if (m_data->m_hasLightDirection)
 			{
-				BT_PROFILE("copy pixels");
+				m_data->m_instancingRenderer->setLightPosition(m_data->m_lightDirection);
-				btAlignedObjectArray<unsigned char> sourceRgbaPixelBuffer;
+			}
-				btAlignedObjectArray<float> sourceDepthBuffer;
+			m_data->m_instancingRenderer->setActiveCamera(&m_data->m_camera);
-				//copy the image into our local cache
+			m_data->m_instancingRenderer->updateCamera(m_data->m_upAxis);
-				sourceRgbaPixelBuffer.resize(sourceWidth * sourceHeight * numBytesPerPixel);
-				sourceDepthBuffer.resize(sourceWidth * sourceHeight);
-				{
-					BT_PROFILE("getScreenPixels");
-					int rgbaBufferSizeInPixels = sourceRgbaPixelBuffer.size();
-					int depthBufferSizeInPixels = sourceDepthBuffer.size();
-					// Copied from SimpleOpenGL3App::getScreenPixels
-					b3Assert((sourceWidth * sourceHeight * 4) == rgbaBufferSizeInPixels);
-					//glClear(GL_COLOR_BUFFER_BIT);
-					//b3Warning("EGL\n");
-					if ((sourceWidth * sourceHeight * 4) == rgbaBufferSizeInPixels)  // remove this if
-					{
-						glReadPixels(0, 0, sourceWidth, sourceHeight, GL_RGBA, GL_UNSIGNED_BYTE, &(sourceRgbaPixelBuffer[0]));
-						int glstat;
-						glstat = glGetError();
-						b3Assert(glstat == GL_NO_ERROR);
-					}
-					if ((sourceWidth * sourceHeight * sizeof(float)) == depthBufferSizeInPixels)
-					{
-						glReadPixels(0, 0, sourceWidth, sourceHeight, GL_DEPTH_COMPONENT, GL_FLOAT, &(sourceDepthBuffer[0]));
-						int glstat;
-						glstat = glGetError();
-						b3Assert(glstat == GL_NO_ERROR);
-					}
-				}
 
-				m_data->m_rgbaPixelBuffer1.resize((*widthPtr) * (*heightPtr) * numBytesPerPixel);
+			m_data->m_instancingRenderer->renderScene();
-				m_data->m_depthBuffer1.resize((*widthPtr) * (*heightPtr));
+
-				//rescale and flip
+			int numBytesPerPixel = 4;  //RGBA
+
+			{
 				{
-					BT_PROFILE("resize and flip");
+					BT_PROFILE("copy pixels");
-					for (int j = 0; j < *heightPtr; j++)
+		
+					//copy the image into our local cache
+					m_data->m_sourceRgbaPixelBuffer.resize(sourceWidth * sourceHeight * numBytesPerPixel);
+					m_data->m_sourceDepthBuffer.resize(sourceWidth * sourceHeight);
 					{
-						for (int i = 0; i < *widthPtr; i++)
+						BT_PROFILE("getScreenPixels");
+						int rgbaBufferSizeInPixels = m_data->m_sourceRgbaPixelBuffer.size();
+						int depthBufferSizeInPixels = m_data->m_sourceDepthBuffer.size();
+						// Copied from SimpleOpenGL3App::getScreenPixels
+						b3Assert((sourceWidth * sourceHeight * 4) == rgbaBufferSizeInPixels);
+						//glClear(GL_COLOR_BUFFER_BIT);
+						//b3Warning("EGL\n");
+						if ((sourceWidth * sourceHeight * 4) == rgbaBufferSizeInPixels)  // remove this if
 						{
-							int xIndex = int(float(i) * (float(sourceWidth) / float(*widthPtr)));
+							glReadPixels(0, 0, sourceWidth, sourceHeight, GL_RGBA, GL_UNSIGNED_BYTE, &(m_data->m_sourceRgbaPixelBuffer[0]));
-							int yIndex = int(float(*heightPtr - 1 - j) * (float(sourceHeight) / float(*heightPtr)));
+							int glstat;
-							btClamp(xIndex, 0, sourceWidth);
+							glstat = glGetError();
-							btClamp(yIndex, 0, sourceHeight);
+							b3Assert(glstat == GL_NO_ERROR);
-							int bytesPerPixel = 4;  //RGBA
+						}
+						if ((sourceWidth * sourceHeight * sizeof(float)) == depthBufferSizeInPixels)
+						{
+							glReadPixels(0, 0, sourceWidth, sourceHeight, GL_DEPTH_COMPONENT, GL_FLOAT, &(m_data->m_sourceDepthBuffer[0]));
+							int glstat;
+							glstat = glGetError();
+							b3Assert(glstat == GL_NO_ERROR);
+						}
+					}
 
-							int sourcePixelIndex = (xIndex + yIndex * sourceWidth) * bytesPerPixel;
+			
-							int sourceDepthIndex = xIndex + yIndex * sourceWidth;
+				}
+			}
+
+	
+			m_data->m_rgbaPixelBuffer1.resize((*widthPtr) * (*heightPtr) * numBytesPerPixel);
+			m_data->m_depthBuffer1.resize((*widthPtr) * (*heightPtr));
+			//rescale and flip
+			{
+				BT_PROFILE("resize and flip");
+				for (int j = 0; j < *heightPtr; j++)
+				{
+					for (int i = 0; i < *widthPtr; i++)
+					{
+						int xIndex = int(float(i) * (float(sourceWidth) / float(*widthPtr)));
+						int yIndex = int(float(*heightPtr - 1 - j) * (float(sourceHeight) / float(*heightPtr)));
+						btClamp(xIndex, 0, sourceWidth);
+						btClamp(yIndex, 0, sourceHeight);
+						int bytesPerPixel = 4;  //RGBA
+
+						int sourcePixelIndex = (xIndex + yIndex * sourceWidth) * bytesPerPixel;
+						int sourceDepthIndex = xIndex + yIndex * sourceWidth;
 #define COPY4PIXELS 1
 #ifdef COPY4PIXELS
-							int* dst = (int*)&m_data->m_rgbaPixelBuffer1[(i + j * (*widthPtr)) * 4 + 0];
+						int* dst = (int*)&m_data->m_rgbaPixelBuffer1[(i + j * (*widthPtr)) * 4 + 0];
-							int* src = (int*)&sourceRgbaPixelBuffer[sourcePixelIndex + 0];
+						int* src = (int*)&m_data->m_sourceRgbaPixelBuffer[sourcePixelIndex + 0];
-							*dst = *src;
+						*dst = *src;
 
 #else
-							m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 0] = sourceRgbaPixelBuffer[sourcePixelIndex + 0];
+						m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 0] = sourceRgbaPixelBuffer[sourcePixelIndex + 0];
-							m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 1] = sourceRgbaPixelBuffer[sourcePixelIndex + 1];
+						m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 1] = sourceRgbaPixelBuffer[sourcePixelIndex + 1];
-							m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 2] = sourceRgbaPixelBuffer[sourcePixelIndex + 2];
+						m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 2] = sourceRgbaPixelBuffer[sourcePixelIndex + 2];
-							m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 3] = 255;
+						m_data->m_rgbaPixelBuffer1[(i + j * widthPtr) * 4 + 3] = 255;
 #endif
-							if (depthBuffer)
+						if (depthBuffer)
+						{
+							m_data->m_depthBuffer1[i + j * (*widthPtr)] = m_data->m_sourceDepthBuffer[sourceDepthIndex];
+						}
+					}
+				}
+			}
+
+			if (segmentationMaskBuffer)
+			{
+				{
+					m_data->m_window->startRendering();
+				
+					BT_PROFILE("renderScene");
+					m_data->m_instancingRenderer->renderSceneInternal(B3_SEGMENTATION_MASK_RENDERMODE);
+				}
+
+				{
+					BT_PROFILE("copy pixels");
+				
+					//copy the image into our local cache
+					m_data->m_segmentationMaskSourceRgbaPixelBuffer.resize(sourceWidth * sourceHeight * numBytesPerPixel);
+					m_data->m_segmentationMaskSourceDepthBuffer.resize(sourceWidth * sourceHeight);
+					{
+						BT_PROFILE("getScreenPixels");
+						{
+							glReadPixels(0, 0, sourceWidth, sourceHeight, GL_DEPTH_COMPONENT, GL_FLOAT, &(m_data->m_segmentationMaskSourceDepthBuffer[0]));
+							int glstat;
+							glstat = glGetError();
+							b3Assert(glstat == GL_NO_ERROR);
+						}
+						{
+							glReadPixels(0, 0, sourceWidth, sourceHeight, GL_RGBA, GL_UNSIGNED_BYTE, &(m_data->m_segmentationMaskSourceRgbaPixelBuffer[0]));
+							int glstat;
+							glstat = glGetError();
+							b3Assert(glstat == GL_NO_ERROR);
+						}
+
+
+					}
+				}
+			}
+
+			
+			m_data->m_segmentationMaskBuffer.resize(destinationWidth * destinationHeight,-1);
+
+			//rescale and flip
+			{
+				BT_PROFILE("resize and flip");
+				for (int j = 0; j < destinationHeight; j++)
+				{
+					for (int i = 0; i < destinationWidth; i++)
+					{
+						int xIndex = int(float(i) * (float(sourceWidth) / float(destinationWidth)));
+						int yIndex = int(float(destinationHeight - 1 - j) * (float(sourceHeight) / float(destinationHeight)));
+						btClamp(xIndex, 0, sourceWidth);
+						btClamp(yIndex, 0, sourceHeight);
+						int bytesPerPixel = 4;  //RGBA
+						int sourcePixelIndex = (xIndex + yIndex * sourceWidth) * bytesPerPixel;
+						int sourceDepthIndex = xIndex + yIndex * sourceWidth;
+
+						if (segmentationMaskBuffer)
+						{
+							float depth = m_data->m_segmentationMaskSourceDepthBuffer[sourceDepthIndex];
+							if (depth<1)
 							{
-								m_data->m_depthBuffer1[i + j * (*widthPtr)] = sourceDepthBuffer[sourceDepthIndex];
+								int segMask = m_data->m_segmentationMaskSourceRgbaPixelBuffer[sourcePixelIndex + 0]+256*(m_data->m_segmentationMaskSourceRgbaPixelBuffer[sourcePixelIndex + 1])+256*256*(m_data->m_segmentationMaskSourceRgbaPixelBuffer[sourcePixelIndex + 2]);
+								m_data->m_segmentationMaskBuffer[i + j * destinationWidth] = segMask;
+							} else
+							{
+								m_data->m_segmentationMaskBuffer[i + j * destinationWidth] = -1;
 							}
 						}
 					}
 				}
 			}
-		}
+			glViewport(0,0,m_data->m_window->getWidth() * m_data->m_window->getRetinaScale(),m_data->m_window->getHeight() * m_data->m_window->getRetinaScale());
 
+		}
 		if (pixelsRGBA)
 		{
 			BT_PROFILE("copy rgba pixels");
@@ -1091,11 +1199,36 @@ void EGLRendererVisualShapeConverter::copyCameraImageDataGL(
 				depthBuffer[i] = m_data->m_depthBuffer1[i + startPixelIndex];
 			}
 		}
+		if (segmentationMaskBuffer)
+		{
+			BT_PROFILE("copy segmentation mask buffer pixels");
+			for (int i = 0; i < numRequestedPixels; i++)
+			{
+
+				int graphicsIndexSegMask = m_data->m_segmentationMaskBuffer[i + startPixelIndex];
+				int segMask = -1;
+				if (graphicsIndexSegMask>=0 && graphicsIndexSegMask< m_data->m_graphicsIndexToSegmentationMask.size())
+				{
+					segMask = m_data->m_graphicsIndexToSegmentationMask[graphicsIndexSegMask];
+				}
+				if ((m_data->m_flags & ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX) == 0)
+				{
+					if (segMask >= 0)
+					{
+						segMask &= ((1 << 24) - 1);
+					}
+				}
+				segmentationMaskBuffer[i] = segMask;
+			}
+		}
+
 		if (numPixelsCopied)
 		{
 			*numPixelsCopied = numRequestedPixels;
 		}
 	}
+
+	
 }
 
 void EGLRendererVisualShapeConverter::copyCameraImageData(unsigned char* pixelsRGBA, int rgbaBufferSizeInPixels,
@@ -1157,6 +1290,7 @@ void EGLRendererVisualShapeConverter::resetAll()
 	m_data->m_textures.clear();
 	m_data->m_swRenderInstances.clear();
 	m_data->m_visualShapes.clear();
+	m_data->m_graphicsIndexToSegmentationMask.clear();
 }
 
 void EGLRendererVisualShapeConverter::changeShapeTexture(int objectUniqueId, int jointIndex, int shapeIndex, int textureUniqueId)

examples/ThirdPartyLibs/imgui/stb_truetype.h

@@ -799,7 +799,7 @@ extern "C"
 	};
 #endif
 
-#ifndef stbtt_vertex             // you can predefine this to use different values \
+#ifndef stbtt_vertex             // you can predefine this to use different values
 								 // (we share this with other code at RAD)
 #define stbtt_vertex_type short  // can't use stbtt_int16 because that's not visible in the header file
 	typedef struct

examples/ThirdPartyLibs/stb_image/stb_truetype.h

@@ -707,7 +707,7 @@ extern "C"
 	};
 #endif
 
-#ifndef stbtt_vertex             // you can predefine this to use different values \
+#ifndef stbtt_vertex             // you can predefine this to use different values
 								 // (we share this with other code at RAD)
 #define stbtt_vertex_type short  // can't use stbtt_int16 because that's not visible in the header file
 	typedef struct

examples/pybullet/pybullet.c

@@ -9800,6 +9800,7 @@ initpybullet(void)
 	PyModule_AddIntConstant(m, "ER_TINY_RENDERER", ER_TINY_RENDERER);
 	PyModule_AddIntConstant(m, "ER_BULLET_HARDWARE_OPENGL", ER_BULLET_HARDWARE_OPENGL);
 	PyModule_AddIntConstant(m, "ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX", ER_SEGMENTATION_MASK_OBJECT_AND_LINKINDEX);
+	PyModule_AddIntConstant(m, "ER_SEGMENTATION_MASK", ER_SEGMENTATION_MASK);
 	PyModule_AddIntConstant(m, "ER_USE_PROJECTIVE_TEXTURE", ER_USE_PROJECTIVE_TEXTURE);
 
 	PyModule_AddIntConstant(m, "IK_DLS", IK_DLS);

setup.py

@@ -559,7 +559,7 @@ if 'BT_USE_EGL' in EGL_CXX_FLAGS:
 
 setup(
 	name = 'pybullet',
-	version='2.2.2',
+	version='2.2.3',
 	description='Official Python Interface for the Bullet Physics SDK specialized for Robotics Simulation and Reinforcement Learning',
 	long_description='pybullet is an easy to use Python module for physics simulation, robotics and deep reinforcement learning based on the Bullet Physics SDK. With pybullet you can load articulated bodies from URDF, SDF and other file formats. pybullet provides forward dynamics simulation, inverse dynamics computation, forward and inverse kinematics and collision detection and ray intersection queries. Aside from physics simulation, pybullet supports to rendering, with a CPU renderer and OpenGL visualization and support for virtual reality headsets.',
 	url='https://github.com/bulletphysics/bullet3',

src/BulletCollision/CollisionDispatch/btCollisionWorldImporter.cpp

@@ -292,9 +292,9 @@ btCollisionShape* btCollisionWorldImporter::convertCollisionShape(btCollisionSha
 			}
 			break;
 		}
-#endif  //SUPPORT_GIMPACT_SHAPE_IMPORT                                                                        \
+#endif      //SUPPORT_GIMPACT_SHAPE_IMPORT
-		//The btCapsuleShape* API has issue passing the margin/scaling/halfextents unmodified through the API \
+			//The btCapsuleShape* API has issue passing the margin/scaling/halfextents unmodified through the API
-		//so deal with this
+			//so deal with this
 		case CAPSULE_SHAPE_PROXYTYPE:
 		{
 			btCapsuleShapeData* capData = (btCapsuleShapeData*)shapeData;

test/GwenOpenGLTest/OpenGLSample.cpp

@@ -347,8 +347,8 @@ int main()
 #endif  // _WIN32
 		gladLoaderLoadGL();
 #endif
-#endif  //B3_USE_GLFW \
+#endif  //B3_USE_GLFW
-	//we ned to call glGetError twice, because of some Ubuntu/Intel/OpenGL issue
+		//we ned to call glGetError twice, because of some Ubuntu/Intel/OpenGL issue
 
 		GLuint err = glGetError();
 		err = glGetError();

test/gtest-1.7.0/src/gtest.cc

@@ -4428,7 +4428,7 @@ UnitTestImpl::UnitTestImpl(UnitTest* parent)
 	: parent_(parent),
 #ifdef _MSC_VER
 #pragma warning(push)            // Saves the current warning state.
-#pragma warning(disable : 4355)  // Temporarily disables warning 4355 \
+#pragma warning(disable : 4355)  // Temporarily disables warning 4355
 								 // (using this in initializer).
 	  default_global_test_part_result_reporter_(this),
 	  default_per_thread_test_part_result_reporter_(this),