Fix two cases of signed integer overflow.

craigcitro · craigcitro · commit 720731083868 · 2017-08-22T15:48:57.000-07:00
This tweaks two cases where there is potential for signed integer overflow: * in `rewind_scanlines`, we switch to unsigned ints * in `not_equal`, we switch for operations which cannot overflow. These cases were caught by matplotlib-using tests being run under [ASan](https://github.com/google/sanitizers) with a check for signed integer overflow.
diff --git a/extern/agg24-svn/include/agg_rasterizer_scanline_aa_nogamma.h b/extern/agg24-svn/include/agg_rasterizer_scanline_aa_nogamma.h
@@ -60,7 +60,7 @@ namespace agg
 
         int not_equal(int ex, int ey, const cell_aa&) const
         {
-            return (ex - x) | (ey - y);
+            return ex != x || ey != y;
         }
     };
 
diff --git a/extern/agg24-svn/include/agg_scanline_storage_aa.h b/extern/agg24-svn/include/agg_scanline_storage_aa.h
@@ -720,10 +720,10 @@ namespace agg
             m_ptr = m_data;
             if(m_ptr < m_end)
             {
-                m_min_x = read_int32() + m_dx; 
-                m_min_y = read_int32() + m_dy;
-                m_max_x = read_int32() + m_dx;
-                m_max_y = read_int32() + m_dy;
+                m_min_x = read_int32u() + m_dx;
+                m_min_y = read_int32u() + m_dy;
+                m_max_x = read_int32u() + m_dx;
+                m_max_y = read_int32u() + m_dy;
             }
             return m_ptr < m_end;
         }

Original file line number	Diff line number	Diff line change
`@@ -60,7 +60,7 @@ namespace agg`
`60`	`60`
`61`	`61`	`int not_equal(int ex, int ey, const cell_aa&) const`
`62`	`62`	`{`
`63`		`- return (ex - x) \| (ey - y);`
	`63`	`+ return ex != x \|\| ey != y;`
`64`	`64`	`}`
`65`	`65`	`};`
`66`	`66`
Original file line number	Diff line number	Diff line change
`@@ -720,10 +720,10 @@ namespace agg`
`720`	`720`	`m_ptr = m_data;`
`721`	`721`	`if(m_ptr < m_end)`
`722`	`722`	`{`
`723`		`- m_min_x = read_int32() + m_dx;`
`724`		`- m_min_y = read_int32() + m_dy;`
`725`		`- m_max_x = read_int32() + m_dx;`
`726`		`- m_max_y = read_int32() + m_dy;`
	`723`	`+ m_min_x = read_int32u() + m_dx;`
	`724`	`+ m_min_y = read_int32u() + m_dy;`
	`725`	`+ m_max_x = read_int32u() + m_dx;`
	`726`	`+ m_max_y = read_int32u() + m_dy;`
`727`	`727`	`}`
`728`	`728`	`return m_ptr < m_end;`
`729`	`729`	`}`