[quartz] work around Quartz bugs with images bigger than a certain size

Quartz has a bug with images greater than (signed) 16 bits in height; avoid
creating or working with those.  Also fixes some memory leaks.

Patch from John Daggett.

src/cairo-quartz-surface.c

@@ -47,6 +47,7 @@
 #endif
 
 #include <Carbon/Carbon.h>
+#include <limits.h>
 
 #undef QUARTZ_DEBUG
 
@@ -107,6 +108,26 @@ extern void CGContextClipToMask (CGContextRef, CGRect, CGImageRef) __attribute__
 static void quartz_surface_to_png (cairo_quartz_surface_t *nq, char *dest);
 static void quartz_image_to_png (CGImageRef, char *dest);
 
+/* CoreGraphics limitation with flipped CTM surfaces: height must be less than signed 16-bit max */
+
+#define CG_MAX_HEIGHT   SHRT_MAX
+#define CG_MAX_WIDTH    USHRT_MAX
+
+/* is the desired size of the surface within bounds? */
+static cairo_bool_t verify_surface_size(int width, int height)
+{
+    /* hmmm, allow width, height == 0 ? */
+    if (width < 0 || height < 0) {
+	return FALSE;
+    }
+
+    if (width > CG_MAX_WIDTH || height > CG_MAX_HEIGHT) {
+	return FALSE;
+    }
+
+    return TRUE;
+}
+
 /*
  * Cairo path -> Quartz path conversion helpers
  */
@@ -466,6 +487,14 @@ _cairo_quartz_surface_to_quartz (cairo_surface_t *target,
         if (status)
 	    return status;
 
+ 	if (new_surf &&
+	    cairo_surface_get_type (new_surf) != CAIRO_SURFACE_TYPE_QUARTZ)
+	{
+	    ND((stderr, "got a non-quartz surface, format=%d width=%u height=%u type=%d\n", cairo_surface_get_type (pat_surf), rect.width, rect.height, cairo_surface_get_type (new_surf)));
+	    cairo_surface_destroy (new_surf);
+	    return CAIRO_INT_STATUS_UNSUPPORTED;
+	}
+
 	*quartz_surf = (cairo_quartz_surface_t *) new_surf;
     } else {
 	/* If it's a quartz surface, we can try to see if it's a CGBitmapContext;
@@ -979,6 +1008,7 @@ _cairo_quartz_surface_release_dest_image (void *abstract_surface,
 
     if (!CGBitmapContextGetData (surface->cgContext)) {
 	CGDataProviderRef dataProvider;
+	CGColorSpaceRef rgb = CGColorSpaceCreateDeviceRGB();
 	CGImageRef img;
 
 	dataProvider = CGDataProviderCreateWithData (NULL, imageData,
@@ -988,12 +1018,13 @@ _cairo_quartz_surface_release_dest_image (void *abstract_surface,
 	img = CGImageCreate (surface->extents.width, surface->extents.height,
 			     8, 32,
 			     surface->extents.width * 4,
-			     CGColorSpaceCreateDeviceRGB(),
+			     rgb,
 			     kCGImageAlphaPremultipliedFirst | kCGBitmapByteOrder32Host,
 			     dataProvider,
 			     NULL,
 			     false,
 			     kCGRenderingIntentDefault);
+	CGColorSpaceRelease (rgb);
 
 	CGContextSetCompositeOperation (surface->cgContext, kPrivateCGCompositeCopy);
 
@@ -1028,6 +1059,12 @@ _cairo_quartz_surface_create_similar (void *abstract_surface,
 	format = CAIRO_FORMAT_A8;
     else
 	return NULL;
+	
+    // verify width and height of surface
+    if (!verify_surface_size(width, height)) {
+	_cairo_error (CAIRO_STATUS_NO_MEMORY);
+	return NULL;
+    }
 
     return cairo_quartz_surface_create (format, width, height);
 }
@@ -1043,6 +1080,13 @@ _cairo_quartz_surface_clone_similar (void *abstract_surface,
 {
     cairo_quartz_surface_t *new_surface = NULL;
     cairo_format_t new_format;
+    
+    *clone_out = NULL;
+
+    // verify width and height of surface
+    if (!verify_surface_size(width, height)) {
+	return CAIRO_INT_STATUS_UNSUPPORTED;
+    }
 
     CGImageRef quartz_image = NULL;
 
@@ -1107,8 +1151,10 @@ _cairo_quartz_surface_clone_similar (void *abstract_surface,
 	cairo_quartz_surface_create (new_format,
 				     CGImageGetWidth (quartz_image),
 				     CGImageGetHeight (quartz_image));
-    if (!new_surface || new_surface->base.status)
+    if (!new_surface || new_surface->base.status) {
+	CGImageRelease (quartz_image);
 	return CAIRO_INT_STATUS_UNSUPPORTED;
+    }
 
     CGContextSetCompositeOperation (new_surface->cgContext,
 				    kPrivateCGCompositeCopy);
@@ -1679,6 +1725,9 @@ static const struct _cairo_surface_backend cairo_quartz_surface_backend = {
 #endif /* CAIRO_HAS_ATSUI_FONT */
 
     NULL, /* snapshot */
+    NULL, /* is_similar */
+    NULL, /* reset */
+    NULL  /* fill_stroke */
 };
 
 static cairo_quartz_surface_t *
@@ -1796,6 +1845,12 @@ cairo_quartz_surface_create (cairo_format_t format,
     int stride;
     int bitsPerComponent;
 
+    // verify width and height of surface
+    if (!verify_surface_size(width, height)) {
+	_cairo_error (CAIRO_STATUS_NO_MEMORY);
+	return (cairo_surface_t*) &_cairo_surface_nil;
+    }
+
     if (format == CAIRO_FORMAT_ARGB32) {
 	cgColorspace = CGColorSpaceCreateDeviceRGB();
 	stride = width * 4;
@@ -1844,6 +1899,7 @@ cairo_quartz_surface_create (cairo_format_t format,
 
     if (!cgc) {
 	_cairo_error (CAIRO_STATUS_NO_MEMORY);
+	free (imageData);
 	return (cairo_surface_t*) &_cairo_surface_nil;
     }
 
@@ -1855,6 +1911,7 @@ cairo_quartz_surface_create (cairo_format_t format,
 						   width, height);
     if (!surf) {
 	CGContextRelease (cgc);
+	free (imageData);
 	// create_internal will have set an error
 	return (cairo_surface_t*) &_cairo_surface_nil;
     }