Fix some color related crashes in libgd

Summary:
Various paths use the color as an index into a 256 element table
without checking it, and passing -7 as a color to imageline would
unconditionally cause it to treat the image as truecolor, seg-faulting
if it wasn't.

Reviewed By: alexmalyshev

Differential Revision: D3658241

fbshipit-source-id: 0f4a9f32dce0a733b5749451b239badd0bfc4d49

Author: jano

hphp/runtime/ext/gd/ext_gd.cpp

@@ -3891,9 +3891,8 @@ Variant HHVM_FUNCTION(imagecolorsforindex, const Resource& image,
                                            int64_t index) {
   gdImagePtr im = get_valid_image_resource(image);
   if (!im) return false;
-  if ((index >= 0 && gdImageTrueColor(im)) ||
-      (!gdImageTrueColor(im) && index >= 0 &&
-       index < gdImageColorsTotal(im))) {
+  if (index >= 0 &&
+      (gdImageTrueColor(im) || index < gdImageColorsTotal(im))) {
     return make_map_array(
       s_red,  gdImageRed(im,index),
       s_green, gdImageGreen(im,index),

hphp/runtime/ext/gd/libgd/gd.cpp

@@ -1011,6 +1011,12 @@ void gdImageAABlend (gdImagePtr im)
 	int p_color, p_red, p_green, p_blue;
 	int px, py;
 
+        if (!gdImageTrueColor(im) && color >= gdImageColorsTotal(im)) {
+          color = gdImageColorsTotal(im) - 1;
+        }
+        if (color < 0) {
+          color = 0;
+        }
 	color_red = gdImageRed(im, color);
 	color_green = gdImageGreen(im, color);
 	color_blue = gdImageBlue(im, color);
@@ -1284,6 +1290,8 @@ inline static void gdImageSetAAPixelColor(gdImagePtr im, int x, int y, int color
  **/
 void gdImageAALine (gdImagePtr im, int x1, int y1, int x2, int y2, int col)
 {
+  if (!gdImageTrueColor(im)) return;
+
 	/* keep them as 32bits */
 	long x, y, inc;
 	long dx, dy,tmp;

hphp/runtime/ext/gd/libgd/gd_crop.cpp

@@ -238,6 +238,10 @@ gdImagePtr gdImageCropThreshold(gdImagePtr im, const unsigned int color,
     return nullptr;
   }
 
+  if (!gdImageTrueColor(im) && color > gdImageColorsTotal(im)) {
+    return nullptr;
+  }
+
   /* TODO: Add gdImageGetRowPtr and works with ptr at the row level
    * for the true color and palette images
    * new formats will simply work with ptr

hphp/runtime/ext/gd/libgd/gd_gd2.cpp

@@ -666,10 +666,12 @@ static void _gdImageGd2 (gdImagePtr im, gdIOCtx * out, int cs, int fmt)
 
 	/* Force fmt to a valid value since we don't return anything. */
 	if ((fmt != GD2_FMT_RAW) && (fmt != GD2_FMT_COMPRESSED)) {
-		fmt = im->trueColor ? GD2_FMT_TRUECOLOR_COMPRESSED : GD2_FMT_COMPRESSED;
+		fmt = GD2_FMT_COMPRESSED;
 	}
 	if (im->trueColor) {
 		fmt += 2;
+                assert(fmt == GD2_FMT_TRUECOLOR_RAW ||
+                       fmt == GD2_FMT_TRUECOLOR_COMPRESSED);
 	}
 	/* Make sure chunk size is valid. These are arbitrary values; 64 because it seems
 	 * a little silly to expect performance improvements on a 64x64 bit scale, and

hphp/runtime/ext/gd/libgd/gd_topal.cpp

@@ -1818,6 +1818,10 @@ static void gdImageTrueColorToPaletteBody (gdImagePtr oim, int dither, int color
     {
       colorsWanted = maxColors;
     }
+  if (colorsWanted <= 0) {
+    colorsWanted = 1;
+  }
+
   if (!cimP) {
     nim->pixels = (unsigned char **) gdCalloc (sizeof (unsigned char *), oim->sy);
     if (!nim->pixels)

hphp/test/slow/ext_gd/colorcrash.php

@@ -0,0 +1,8 @@
+<?php
+$img=imagecreatetruecolor(10, 10);
+imagetruecolortopalette($img, false, PHP_INT_MAX / 8);
+
+$img = imagecreate(100, 100);
+imageline($img, 0, 0, 100, 100, -7);
+imagepolygon($img, array(10,10,10,50,50,50,50,10,10,10), 5, -7);
+echo "done\n";

hphp/test/slow/ext_gd/colorcrash.php.expect

@@ -0,0 +1 @@
+done