http://openwall.com/lists/oss-security/2017/08/31/1
http://openwall.com/lists/oss-security/2017/08/31/2
http://hg.code.sf.net/p/graphicsmagick/code/raw-rev/233a720bfd5e

some changes were made to make the patch apply

# HG changeset patch
# User Bob Friesenhahn <bfriesen@GraphicsMagick.org>
# Date 1503779175 18000
# Node ID 233a720bfd5efd378f133a776507ed41230da617
# Parent  b037d79b6ccd0cfba7ba9ce09b454ed46d688036
XBM: Fix DOS issues.

diff -r b037d79b6ccd -r 233a720bfd5e coders/xbm.c
--- a/coders/xbm.c	Sat Aug 26 14:14:13 2017 -0500
+++ b/coders/xbm.c	Sat Aug 26 15:26:15 2017 -0500
@@ -1,5 +1,5 @@
 /*
-% Copyright (C) 2003 -2012 GraphicsMagick Group
+% Copyright (C) 2003-2017 GraphicsMagick Group
 % Copyright (C) 2002 ImageMagick Studio
 % Copyright 1991-1999 E. I. du Pont de Nemours and Company
 %
@@ -121,13 +121,15 @@
 
 static int XBMInteger(Image *image,short int *hex_digits)
 {
+  unsigned int
+    flag;
+
   int
     c,
-    flag,
     value;
 
   value=0;
-  flag=0;
+  flag=0U;
   for ( ; ; )
   {
     c=ReadBlobByte(image);
@@ -158,18 +160,14 @@
   Image
     *image;
 
-  int
-    bit;
-
-  long
-    y;
-
   register IndexPacket
     *indexes;
 
-  register long
+  register size_t
+    bytes_per_line,
     i,
-    x;
+    x,
+    y;
 
   register PixelPacket
     *q;
@@ -177,22 +175,24 @@
   register unsigned char
     *p;
 
-  short int
-    hex_digits[256];
-
   unsigned char
     *data;
 
   unsigned int
+    bit,
+    byte,
+    padding,
+    version;
+
+  int
+    value;
+
+  short int
+    hex_digits[256];
+
+  MagickPassFail
     status;
 
-  unsigned long
-    byte,
-    bytes_per_line,
-    padding,
-    value,
-    version;
-
   /*
     Open image file.
   */
@@ -207,6 +207,8 @@
   /*
     Read X bitmap header.
   */
+  (void) memset(buffer,0,sizeof(buffer));
+  name[0]='\0';
   while (ReadBlobString(image,buffer) != (char *) NULL)
     if (sscanf(buffer,"#define %s %lu",name,&image->columns) == 2)
       if ((strlen(name) >= 6) &&
@@ -278,6 +280,8 @@
   /*
     Initialize hex values.
   */
+  for (i = 0; i < sizeof(hex_digits)/sizeof(hex_digits[0]); i++)
+    hex_digits[i]=(-1);
   hex_digits['0']=0;
   hex_digits['1']=1;
   hex_digits['2']=2;
@@ -311,40 +315,50 @@
   */
   p=data;
   if (version == 10)
-    for (i=0; i < (long) (bytes_per_line*image->rows); (i+=2))
+    for (i=0; i < (bytes_per_line*image->rows); (i+=2))
     {
       value=XBMInteger(image,hex_digits);
+      if (value < 0)
+        {
+          MagickFreeMemory(data);
+          ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+        }
       *p++=(unsigned char) value;
       if (!padding || ((i+2) % bytes_per_line))
         *p++=(unsigned char) (value >> 8);
     }
   else
-    for (i=0; i < (long) (bytes_per_line*image->rows); i++)
+    for (i=0; i < (bytes_per_line*image->rows); i++)
     {
       value=XBMInteger(image,hex_digits);
+      if (value < 0)
+        {
+          MagickFreeMemory(data);
+          ThrowReaderException(CorruptImageError,ImproperImageHeader,image);
+        }
       *p++=(unsigned char) value;
     }
   /*
     Convert X bitmap image to pixel packets.
   */
   p=data;
-  for (y=0; y < (long) image->rows; y++)
+  for (y=0; y < image->rows; y++)
   {
     q=SetImagePixels(image,0,y,image->columns,1);
     if (q == (PixelPacket *) NULL)
       break;
     indexes=AccessMutableIndexes(image);
-    bit=0;
-    byte=0;
-    for (x=0; x < (long) image->columns; x++)
+    bit=0U;
+    byte=0U;
+    for (x=0; x < image->columns; x++)
     {
-      if (bit == 0)
+      if (bit == 0U)
         byte=(*p++);
       indexes[x]=byte & 0x01 ? 0x01 : 0x00;
       bit++;
-      byte>>=1;
-      if (bit == 8)
-        bit=0;
+      byte>>=1U;
+      if (bit == 8U)
+        bit=0U;
     }
     if (!SyncImagePixels(image))
       break;