mex out of memory

Matlab would be out of memory, if i use a mex file in a loop. I think it's caused by memory leak. mxMalloc variable is freed by mxFree, but I cannot destroy mxCreateNumericArray variable using mxDestroyArray(plhs[0]). Any help would be appreciated!

The mex file is from Offscreen toolbox http://www.mathworks.de/matlabcentral/fileexchange/25071-matlab-offscreen-rendering-toolbox . The code is as following.

#include "mex.h"
#include "math.h"
#include "OffscreenGL.h"
#include "OffscreenCommon.h"

void drawPatchAndConvert(GLuint listName, GLubyte *imageBuffer, unsigned int imgHeight, unsigned int imgWidth, unsigned int zoomFactor = 1)
{
  // This is a temporary bug fix for Nvidia's open program
  // seems the width of the pixel has to be a multiple of 4
  // for other width, we have to pad the width and remove it later
  unsigned int paddedWidth = imgWidth * zoomFactor % 4;
  if (paddedWidth != 0) {
    paddedWidth = 4 - paddedWidth + imgWidth * zoomFactor;
  } else {
    paddedWidth = imgWidth * zoomFactor;
  }
  
  unsigned char *paddedImgBuffer = (unsigned char *)mxMalloc(paddedWidth * imgHeight * zoomFactor * MAX_COLOR_CHANNEL * sizeof(GL_UNSIGNED_BYTE));
  drawPatch(listName, paddedImgBuffer, imgHeight, imgWidth, zoomFactor);  

  // reorder the pixel data for the opengl to matlab conversion
  unsigned int imgSize = imgHeight * imgWidth * zoomFactor * zoomFactor;
  unsigned int imgSize2 = imgSize * 2;
  unsigned int matlabImgIndex = 0;
  unsigned int oglImageIndex = 0;
  
  for (int j = 0; j < imgWidth * zoomFactor; j++) {
    for (int i = 0; i < imgHeight * zoomFactor; i++, matlabImgIndex++) {
      oglImageIndex = (j + (imgHeight*zoomFactor -1-i) * paddedWidth) * 3;
        imageBuffer[matlabImgIndex] = paddedImgBuffer[oglImageIndex];
        imageBuffer[matlabImgIndex + imgSize] = paddedImgBuffer[oglImageIndex + 1];
        imageBuffer[matlabImgIndex + imgSize2] = paddedImgBuffer[oglImageIndex + 2];
    }
  }
    mxFree(paddedImgBuffer);

}

static void renderColorMesh(double *FM, int fNum, double *VM, int vNum, float *ColorM, int colorNum,
          const mxArray *CamParamS, double *imgSizeV, double *zNearFarV, unsigned int zoomFactor,
          // output
          unsigned char *imgBuffer)
{
  cameraSetup(CamParamS, zNearFarV[0], zNearFarV[1], (unsigned int) imgSizeV[0], (unsigned int) imgSizeV[1], zoomFactor);

#ifndef NDEBUG
  mexPrintf("Start to create the display list: fNum=%d, vNum=%d, colorNum=%d\n", fNum, vNum, colorNum);
#endif

  GLuint list = createDisplayListWithColor(FM, fNum, VM, vNum, ColorM, colorNum);

#ifndef NDEBUG
  mexPrintf("Start to draw the patch\n");
#endif

  drawPatchAndConvert(list, imgBuffer, (int) imgSizeV[0], (int) imgSizeV[1], zoomFactor);
}

void mexFunction(int nlhs, mxArray *plhs[],
                 int nrhs, const mxArray *prhs[])
{
  // get the vertex array, face array, and color array
  double *FM = mxGetPr(prhs[0]);
  int fNum = mxGetM(prhs[0]);  
  double *VM = mxGetPr(prhs[1]);
  int vNum = mxGetM(prhs[1]);
  float *ColorM = (float *)mxGetData(prhs[2]);
  int colorNum = mxGetM(prhs[2]);
  
  // get the camera parameters
  const mxArray *CamParamS = prhs[3];
  double *imgSizeV = mxGetPr(prhs[4]);
  double *zNearFarV = mxGetPr(prhs[5]);
  double zoomFactor = mxGetScalar(prhs[6]);
  
  OffscreenGL offscreenGL((int)(imgSizeV[0] * zoomFactor), (int) (imgSizeV[1] * zoomFactor));
  int output3Size[3];
  
  unsigned char *imgBuffer;
  
  if (offscreenGL.RGB8Setup()) {
    //mexPrintf("OpenGLCanvas setup Successful\n");
    output3Size[0] = (int) (imgSizeV[0] * zoomFactor);
    output3Size[1] = (int) (imgSizeV[1] * zoomFactor);
    output3Size[2] = 3;
    
        plhs[0] = mxCreateNumericArray(3, output3Size, mxUINT8_CLASS, mxREAL);
      imgBuffer = (unsigned char *) mxGetData(plhs[0]);
    renderColorMesh(FM, fNum, VM, vNum, ColorM, colorNum, CamParamS, imgSizeV,
            zNearFarV, (unsigned int) zoomFactor, imgBuffer);
    
  } else {
    mexPrintf("OpenGLCanvas setup failed\n");    
  }
}

Answer 1

[Jan]

mxCreateNumericArray requires the dimensions to by a mwSize array. This could differ from the provided int for 64 bit compilers.

You should not free plhs[0], because this is returned to the caller. I'm not convinced that there is a problem in the MEX at all. Perhaps you collect the outputs iteratively in the Matlab side. Then e.g. a forgotten pre-allocation could exhaust your memory also.

[nt]

Thanks for your fast reply. You are right that it is not because of this MEX file. It is because of a function used by this file.

GLuint list = createDisplayListWithColor(FM, fNum, VM, vNum, ColorM, colorNum);

It is the problem of openGL as following.

static GLuint createDisplayListWithColor(double *fM, int fNum, double *vM, int vNum, GLfloat *ColorM, int colorNum)
{
  GLuint theShape;

  int i;
  double *fp;
  int vIndex, fNum2, vNum2;
  
  fNum2 = fNum * 2;
  vNum2 = vNum * 2;
  
  theShape = glGenLists (1);
  glNewList(theShape, GL_COMPILE);
  glBegin (GL_TRIANGLES);

  if (colorNum == vNum) { // vertex color
    for (i = 1; i <= fNum; i++) {
      fp = fM + i-1;
      vIndex = (int)fp[0] - 1;
      
      glColor3f(ColorM[vIndex], ColorM[vIndex + vNum], ColorM[vIndex + vNum2]);//<- it causes memory leak 
      /* 
      xxxxxxx
      */
    }
  }
}

[Jan]

Please do not cross-post. I wastes the time of the ones, who want to help you. If you have really good reasons to post a question in different forums, please add a link to the other forum also. Thanks.