Fix segfault with antialiased draw functions with a depth different t…

…han 32bits (pygame-community#3008) * fix segfault * format fix * format issue * another format issue * f issue * add tests * py format fix * removed 24bits case * removed masks * format fix * fix segfault & antialiasing for 24bits * format issue * format issue 2 * format issue 3 * switch refactor * clang format * draw.c skill issue * test update * refactor * 8bit fix * format issue * big endian fix * switch case * hopefully it fixes some problems * removed not necessary include * SDL3 compatibility * size_t to int
damusss · Sep 24, 2024 · faef6d4 · faef6d4
1 parent c0d17a4
commit faef6d4
Show file tree

Hide file tree

Showing 2 changed files with 74 additions and 2 deletions.
diff --git a/src_c/draw.c b/src_c/draw.c
@@ -1126,15 +1126,42 @@ get_antialiased_color(SDL_Surface *surf, int x, int y, Uint32 original_color,
                       float brightness)
 {
     Uint8 color_part[4], background_color[4];
-    Uint32 *pixels = (Uint32 *)surf->pixels;
     SDL_GetRGBA(original_color, surf->format, &color_part[0], &color_part[1],
                 &color_part[2], &color_part[3]);
     if (x < surf->clip_rect.x || x >= surf->clip_rect.x + surf->clip_rect.w ||
         y < surf->clip_rect.y || y >= surf->clip_rect.y + surf->clip_rect.h)
         return original_color;
-    SDL_GetRGBA(pixels[(y * surf->w) + x], surf->format, &background_color[0],
+
+    Uint32 pixel = 0;
+    int bpp = PG_SURF_BytesPerPixel(surf);
+    Uint8 *pixels = (Uint8 *)surf->pixels + y * surf->pitch + x * bpp;
+
+    switch (bpp) {
+        case 1:
+            pixel = *pixels;
+            break;
+
+        case 2:
+            pixel = *((Uint16 *)pixels);
+            break;
+
+        case 3:
+#if SDL_BYTEORDER == SDL_LIL_ENDIAN
+            pixel = (pixels[0]) + (pixels[1] << 8) + (pixels[2] << 16);
+#else  /* SDL_BIG_ENDIAN */
+            pixel = (pixels[2]) + (pixels[1] << 8) + (pixels[0] << 16);
+#endif /* SDL_BIG_ENDIAN */
+            break;
+
+        default: /* case 4: */
+            pixel = *((Uint32 *)pixels);
+            break;
+    }
+
+    SDL_GetRGBA(pixel, surf->format, &background_color[0],
                 &background_color[1], &background_color[2],
                 &background_color[3]);
+
     color_part[0] = (Uint8)(brightness * color_part[0] +
                             (1 - brightness) * background_color[0]);
     color_part[1] = (Uint8)(brightness * color_part[1] +

diff --git a/test/draw_test.py b/test/draw_test.py
@@ -7176,6 +7176,51 @@ def test_color_validation(self):
             with self.assertRaises(TypeError):
                 draw.polygon(surf, col, points, 0)
 
+    def test_aafunctions_depth_segfault(self):
+        """Ensure future commits don't break the segfault fixed by pull request
+        https://github.com/pygame-community/pygame-ce/pull/3008
+        """
+
+        pixels_to_check = [(160, 102), (499, 300), (320, 258), (192, 252)]
+        pixel_colors_8 = [
+            pygame.Color(0, 182, 0, 255),
+            pygame.Color(0, 0, 170, 255),
+            pygame.Color(255, 0, 0, 255),
+            pygame.Color(255, 0, 0, 255),
+        ]
+
+        pixel_colors_16 = [
+            pygame.Color(0, 178, 0, 255),
+            pygame.Color(0, 0, 213, 255),
+            pygame.Color(246, 0, 0, 255),
+            pygame.Color(222, 0, 0, 255),
+        ]
+
+        pixel_colors_24_32 = [
+            pygame.Color(0, 178, 0, 255),
+            pygame.Color(0, 0, 209, 255),
+            pygame.Color(247, 0, 0, 255),
+            pygame.Color(223, 0, 0, 255),
+        ]
+
+        for depth in (8, 16, 24, 32):
+            # all values must stay so to reproduce the segfault
+            surf = pygame.Surface(size=(512, 512), flags=0, depth=depth)
+
+            draw.aacircle(surf, pygame.Color("red"), (256, 256), 64)
+            draw.aaline(surf, pygame.Color("blue"), (256, 256), (500, 300))
+            draw.aalines(
+                surf, pygame.Color("green"), False, [(100, 100), (100, 500), (160, 100)]
+            )
+
+            for i, pixel in enumerate(pixels_to_check):
+                if depth == 8:
+                    self.assertEqual(surf.get_at(pixel), pixel_colors_8[i])
+                elif depth == 16:
+                    self.assertEqual(surf.get_at(pixel), pixel_colors_16[i])
+                else:
+                    self.assertEqual(surf.get_at(pixel), pixel_colors_24_32[i])
+
 
 ###############################################################################