Image Processing Examples

Standalone ExamplesExternal Library Examples

The Wolfram Language makes it possible to interface with existing image processing libraries efficiently using Wolfram LibraryLink. By interfacing to libraries, you can make use of existing code from within the Wolfram Language. This tutorial walks you through some examples using LibraryLink along with image processing.

LibraryLink provides a way to interface the Wolfram Language to C or C++ code. The interface is low level but efficient, and is targeted to users who wish to either use existing native code or develop C routines for computationally expensive parts of image processing applications. The suggested flow of development is: develop your application using the Wolfram Language, identify bottlenecks, and then implement those routines in C. This tutorial follows this advice: the Wolfram Language implementation will be introduced, then the routine will be implemented in C and compared against the Wolfram Language.

The source for the examples shown in this tutorial is found in the documentation paclet.

demo_image.cxxsource code for standalone image processing examples
image_external.csource code for image processing examples interfacing to external libraries
image_video.cxxsource code for image processing examples interfacing to external video library

The source files can be found in the following directory.

Standalone Examples

Two external libraries are used in this code: OpenCV and LibRaw. These libraries are not provided with the Wolfram Language, so in order to compile and run the code, they need to be installed on your machine.

Using Templates to Parametrize on Type

Since Wolfram Language images support different types ("Bit", "Byte", "Bit16", "Real32", and "Real"), templates can be used to both simplify your code and make it more maintainable. In color_negate, you can parametrize on the type and avoid reimplementing the same function multiple times, as shown below.

template <typename T> static T maxValue() {
  return -1; // ERROR
}

template <> char maxValue<char>() { return 1; }

template <> raw_t_ubit8 maxValue<raw_t_ubit8>() { return 255; }

template <> raw_t_ubit16 maxValue<raw_t_ubit16>() { return 65535; }

template <> raw_t_real32 maxValue<raw_t_real32>() { return 1.0f; }

template <> raw_t_real64 maxValue<raw_t_real64>() { return 1.0; }

template <typename T>
static void icolor_negate(void *out0, const void *in0, mint length) {
  mint ii;
  T *out = reinterpret_cast<T *>(out0);
  const T *in = reinterpret_cast<const T *>(in0);
  for (ii = 0; ii < length; ii++) {
    out[ii] = maxValue<T>() - in[ii];
  }
}

/* Negate image colors */
EXTERN_C DLLEXPORT int color_negate(WolframLibraryData libData, mint Argc,
                                    MArgument *Args, MArgument res) {
  mint length;
  MImage image_in, image_out = 0;
  void *data_in, *data_out;
  int err = LIBRARY_FUNCTION_ERROR;
  imagedata_t type;
  WolframImageLibrary_Functions imgFuns = libData->imageLibraryFunctions;

  if (Argc < 1) {
    return err;
  }

  image_in = MArgument_getMImage(Args[0]);

  err = imgFuns->MImage_clone(image_in, &image_out);
  if (err)
    return err;

  type = imgFuns->MImage_getDataType(image_in);
  length = imgFuns->MImage_getFlattenedLength(image_in);

  data_in = imgFuns->MImage_getRawData(image_in);
  data_out = imgFuns->MImage_getRawData(image_out);
  if (data_in == NULL || data_out == NULL)
    goto cleanup;

  switch (type) {
  case MImage_Type_Bit:
    icolor_negate<char>(data_out, data_in, length);
    break;
  case MImage_Type_Bit8:
    icolor_negate<raw_t_ubit8>(data_out, data_in, length);
    break;
  case MImage_Type_Bit16:
    icolor_negate<raw_t_ubit16>(data_out, data_in, length);
    break;
  case MImage_Type_Real32:
    icolor_negate<raw_t_real32>(data_out, data_in, length);
    break;
  case MImage_Type_Real:
    icolor_negate<raw_t_real64>(data_out, data_in, length);
    break;
  default:
    goto cleanup;
  }

  MArgument_setMImage(res, image_out);
  return err;

cleanup:
  imgFuns->MImage_free(image_out);
  return err;
}

You can use the code as follows.

Load the color_negate function from the library.
Here a set of images that has different types is defined.
Now the color_negate function is applied on the images.

Restricting Accepted Types

In some cases, the algorithm only works on certain data types or it becomes too verbose to convert to the different data types from within C. In those cases, you need to convert the image types to the ones supported by the C code. This can be done automatically by the LibraryFunction call. Consider the special function defined below.

static void isepia(raw_t_real32 *out, raw_t_real32 *in, mint width, mint height,
                   mint channels) {
  for (mint ii = 0; ii < height; ii++) {
    for (mint jj = 0; jj < width; jj++) {
      for (mint kk = 0; kk < channels; kk++) {
        mint index = channels * (ii * width + jj);
        raw_t_real32 r = in[index + 0];
        raw_t_real32 g = in[index + 1];
        raw_t_real32 b = in[index + 2];

        out[index + 0] = r * static_cast<raw_t_real32>(0.393) +
                         g * static_cast<raw_t_real32>(0.769) +
                         b * static_cast<raw_t_real32>(0.189);
        out[index + 1] = r * static_cast<raw_t_real32>(0.349) +
                         g * static_cast<raw_t_real32>(0.686) +
                         b * static_cast<raw_t_real32>(0.168);
        out[index + 2] = r * static_cast<raw_t_real32>(0.272) +
                         g * static_cast<raw_t_real32>(0.534) +
                         b * static_cast<raw_t_real32>(0.131);

        for (int ii = 3; ii < channels; ii++) {
          out[index + ii] = in[index + ii];
        }
      }
    }
  }
  return;
}


EXTERN_C DLLEXPORT int speia(WolframLibraryData libData, mint Argc,
    MArgument *Args, MArgument res) {
    mbool alphaQ;
    int err = 0;
    imagedata_t type;
    colorspace_t cs;
    MImage image_in, image_out;
    mint height, width, channels;
    raw_t_real32 *data_in, *data_out;
    WolframImageLibrary_Functions imgFuns = libData->imageLibraryFunctions;

    if (Argc != 1) {
        return LIBRARY_FUNCTION_ERROR;
    }

    image_in = MArgument_getMImage(Args[0]);
    if (imgFuns->MImage_getColorSpace(image_in) != MImage_CS_RGB)
        return LIBRARY_FUNCTION_ERROR;

    if (imgFuns->MImage_getRank(image_in) == 3)
        return LIBRARY_RANK_ERROR;

    type = imgFuns->MImage_getDataType(image_in);
    height = imgFuns->MImage_getRowCount(image_in);
    width = imgFuns->MImage_getColumnCount(image_in);
    channels = imgFuns->MImage_getChannels(image_in);
    cs = imgFuns->MImage_getColorSpace(image_in);
    alphaQ = imgFuns->MImage_alphaChannelQ(image_in);

    if (type != MImage_Type_Real32)
        return LIBRARY_TYPE_ERROR;

    err = imgFuns->MImage_new2D(width, height, channels, type, cs, True,
        &image_out);
    if (err)
        return LIBRARY_FUNCTION_ERROR;

    data_in = imgFuns->MImage_getReal32Data(image_in);
    data_out = imgFuns->MImage_getReal32Data(image_out);
    if (data_in == NULL || data_out == NULL)
        return LIBRARY_FUNCTION_ERROR;

    ispeia(data_out, data_in, width, height, channels);

    MArgument_setMImage(res, image_out);
    return LIBRARY_NO_ERROR;
}
Load the special function from the library, autoconverting to the "Real32" type.
Now the special function is applied on the images defined earlier.

Sharing Images

Sharing image references between the Wolfram Language kernel and your C code can be an important optimization, since it decreases memory usage. On the other hand, it does place more code complication in your C code and there is a risk of memory leaks.

External Library Examples

Two external libraries are used in this code: OpenCV and LibRaw. These libraries are not provided with the Wolfram Language, so in order to compile and run the code, they need to be installed on your machine.

Needs["LibraryLink`"]
Needs["CCompilerDriver`"]
opencvDir = "c:\\opencv";
opencvIncludeDir=FileNameJoin[{opencvDir,"build","include",#}]&/@{"","opencv","opencv2", "opencv2\\imgproc"};
opencvLibDir=FileNameJoin[{opencvDir,"build","x64","vc11",#}]&/@{"bin","lib"};
opencvLibraries={"opencv_core246","opencv_highgui246","opencv_imgproc246"};
librawDir="c:\\libraw";
librawIncludeDir=FileNameJoin[{librawDir,"libraw"}];
librawLibDir=FileNameJoin[{librawDir,"build"}];
librawLibraries={"libraw"};
lib=CreateLibrary[{FileNameJoin[{sourcePath, "image_external.c"}]},"image_external","IncludeDirectories"Append[opencvIncludeDir,librawIncludeDir],"LibraryDirectories"Append[opencvLibDir,librawLibDir],"Libraries"Join[opencvLibraries,librawLibraries],"Defines""WIN32"];
LibraryLoad[FileNameJoin[{opencvDir,"build","x64","vc11","bin",#}]]&/@{"opencv_core246","opencv_highgui246","opencv_imgproc246"};
LibraryLoad[FileNameJoin[{librawLibDir,"libraw"}]];

OpenCV Example

Following is an example that implements a morphological dilation operation using OpenCV library available at http://opencv.org. Function opencv_dilate takes two arguments: a grayscale 2D image and a radius that defines a structuring element in the form of a square . Since OpenCV does not support 64-bit floating point images, IPL_DEPTH_32F is used when the input image is of the type MImage_Type_Real.

This shows the source code.
This loads the function.
Once the function is loaded, it can be used as an ordinary function.
Compare with the built-in Dilation function.

Importing Frames from a Video

FFMPEG (http://www.ffmpeg.org) is a library for reading videos of a variety of formats. The following shows how to implement a basic video player from within the Wolfram Language.

The link allows you to open a file, register it as a managed library expression, and then control the position within the video and output the image frame.

This shows the source code.
You need to load the required dependent libraries.
This loads the functions.
If you load a video, you get a VideoInstance.
You can then open the file and bind the internal data to that instance.
You can then get the next frame in the video.

Importing Raw Images

LibRaw (http://www.libraw.org) is a library for reading RAW files obtained from digital photo cameras (CRW/CR2, NEF, RAF, DNG, and others). Here is how to use it to import raw images into the Wolfram Language.

The function read_raw_image shown below takes a path to a file and returns an image expression.

EXTERN_C DLLEXPORT int read_raw_image(WolframLibraryData libData, mint Argc, MArgument *Args, MArgument res) {
    int err;
    int check;
    MImage out;
    char * file;
    libraw_data_t *iprc = libraw_init(0);
    libraw_processed_image_t * img;
    WolframImageLibrary_Functions imgFuns = libData->imageLibraryFunctions;

    err = LIBRARY_FUNCTION_ERROR;
    file = MArgument_getUTF8String(Args[0]);

    libraw_open_file(iprc, file);
    libraw_unpack(iprc);

    iprc->params.output_bps = 8;

    check = libraw_dcraw_process(iprc);
    if (check != LIBRAW_SUCCESS) goto cleanup;

    img = libraw_dcraw_make_mem_image(iprc, &check);
    if (img == NULL) goto cleanup;
    if (img->type != LIBRAW_IMAGE_BITMAP || img->colors != 3) goto cleanup;
    
    if (img->bits == 16) {
        raw_t_ubit16 * raw_data = (raw_t_ubit16*)img->data;
        imgFuns->MImage_new2D(
            img->width, img->height, 3,
            MImage_Type_Bit16, MImage_CS_RGB, 1,
            &out);
        memcpy(imgFuns->MImage_getBit16Data(out),
               raw_data,
               img->width * img->height * 3 * sizeof(raw_t_ubit16));
    } else if (img->bits == 8) {
        raw_t_ubit8 * raw_data = (raw_t_ubit8*)img->data;
        imgFuns->MImage_new2D(img->width, img->height, 3,
                              MImage_Type_Bit8, MImage_CS_RGB, 1,
                              &out);
        memcpy(imgFuns->MImage_getByteData(out),
               raw_data,
               img->width * img->height * 3 * sizeof(raw_t_ubit8));
    } else {
        goto cleanup;
    }
    
    MArgument_setMImage(res, out);
    err = LIBRARY_NO_ERROR;

cleanup:
    libData->UTF8String_disown(file);
    libraw_dcraw_clear_mem(img);
    return err;
}
Load the function.
Obtain a raw file.
Import the file.