diff mbox series

[3/3] drm/gud: Fix endianness in gud_xrgb8888_to_color() helper

Message ID b47589ed5d8ca44e0956684412e3f16f3227f887.1657300532.git.geert@linux-m68k.org
State New
Headers show
Series drm: Endianness fixes | expand

Commit Message

Geert Uytterhoeven July 8, 2022, 6:21 p.m. UTC
DRM formats are defined to be little-endian, unless the
DRM_FORMAT_BIG_ENDIAN flag is set.  Hence when converting from one
format to another, multi-byte pixel values loaded from memory must be
converted from little-endian to host-endian.  Conversely, multi-byte
pixel values written to memory must be converted from host-endian to
little-endian.  Currently only drm_fb_xrgb8888_to_rgb332_line() includes
endianness handling.

Fix gud_xrgb8888_to_color() on big-endian platforms by adding the
missing endianness handling.

Signed-off-by: Geert Uytterhoeven <geert@linux-m68k.org>
---
Compile-tested only.

Interestingly, drm_fb_xrgb8888_to_rgb332() was introduced for GUD,
and always had correct endiannes handling...
---
 drivers/gpu/drm/gud/gud_pipe.c | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

Comments

Noralf Trønnes July 19, 2022, 2:39 p.m. UTC | #1
Den 08.07.2022 20.21, skrev Geert Uytterhoeven:
> DRM formats are defined to be little-endian, unless the
> DRM_FORMAT_BIG_ENDIAN flag is set.  Hence when converting from one
> format to another, multi-byte pixel values loaded from memory must be
> converted from little-endian to host-endian.  Conversely, multi-byte
> pixel values written to memory must be converted from host-endian to
> little-endian.  Currently only drm_fb_xrgb8888_to_rgb332_line() includes
> endianness handling.
> 
> Fix gud_xrgb8888_to_color() on big-endian platforms by adding the
> missing endianness handling.
> 
> Signed-off-by: Geert Uytterhoeven <geert@linux-m68k.org>
> ---
> Compile-tested only.
> 
> Interestingly, drm_fb_xrgb8888_to_rgb332() was introduced for GUD,
> and always had correct endiannes handling...

RGB332 support was added later and by that time I had understood that
the framebuffer was little endian and not host endian as I first assumed
(there's a fixme comment in gud_pipe.c that BE is probably broken but I
haven't got any hw to test on so I haven't tried to fix it).

Thanks for fixing this, pathces 2 and 3 tested on drm/gud and applied to
drm-misc-next.

Noralf.

> ---
>  drivers/gpu/drm/gud/gud_pipe.c | 14 ++++++++------
>  1 file changed, 8 insertions(+), 6 deletions(-)
> 
> diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c
> index 4873f9799f412e04..d42592f6daab8b2a 100644
> --- a/drivers/gpu/drm/gud/gud_pipe.c
> +++ b/drivers/gpu/drm/gud/gud_pipe.c
> @@ -105,7 +105,8 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
>  	unsigned int bits_per_pixel = 8 / block_width;
>  	u8 r, g, b, pix, *block = dst; /* Assign to silence compiler warning */
>  	unsigned int x, y, width;
> -	u32 *pix32;
> +	__le32 *sbuf32;
> +	u32 pix32;
>  	size_t len;
>  
>  	/* Start on a byte boundary */
> @@ -114,8 +115,8 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
>  	len = drm_format_info_min_pitch(format, 0, width) * drm_rect_height(rect);
>  
>  	for (y = rect->y1; y < rect->y2; y++) {
> -		pix32 = src + (y * fb->pitches[0]);
> -		pix32 += rect->x1;
> +		sbuf32 = src + (y * fb->pitches[0]);
> +		sbuf32 += rect->x1;
>  
>  		for (x = 0; x < width; x++) {
>  			unsigned int pixpos = x % block_width; /* within byte from the left */
> @@ -126,9 +127,10 @@ static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
>  				*block = 0;
>  			}
>  
> -			r = *pix32 >> 16;
> -			g = *pix32 >> 8;
> -			b = *pix32++;
> +			pix32 = le32_to_cpu(*sbuf32++);
> +			r = pix32 >> 16;
> +			g = pix32 >> 8;
> +			b = pix32;
>  
>  			switch (format->format) {
>  			case GUD_DRM_FORMAT_XRGB1111:
diff mbox series

Patch

diff --git a/drivers/gpu/drm/gud/gud_pipe.c b/drivers/gpu/drm/gud/gud_pipe.c
index 4873f9799f412e04..d42592f6daab8b2a 100644
--- a/drivers/gpu/drm/gud/gud_pipe.c
+++ b/drivers/gpu/drm/gud/gud_pipe.c
@@ -105,7 +105,8 @@  static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 	unsigned int bits_per_pixel = 8 / block_width;
 	u8 r, g, b, pix, *block = dst; /* Assign to silence compiler warning */
 	unsigned int x, y, width;
-	u32 *pix32;
+	__le32 *sbuf32;
+	u32 pix32;
 	size_t len;
 
 	/* Start on a byte boundary */
@@ -114,8 +115,8 @@  static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 	len = drm_format_info_min_pitch(format, 0, width) * drm_rect_height(rect);
 
 	for (y = rect->y1; y < rect->y2; y++) {
-		pix32 = src + (y * fb->pitches[0]);
-		pix32 += rect->x1;
+		sbuf32 = src + (y * fb->pitches[0]);
+		sbuf32 += rect->x1;
 
 		for (x = 0; x < width; x++) {
 			unsigned int pixpos = x % block_width; /* within byte from the left */
@@ -126,9 +127,10 @@  static size_t gud_xrgb8888_to_color(u8 *dst, const struct drm_format_info *forma
 				*block = 0;
 			}
 
-			r = *pix32 >> 16;
-			g = *pix32 >> 8;
-			b = *pix32++;
+			pix32 = le32_to_cpu(*sbuf32++);
+			r = pix32 >> 16;
+			g = pix32 >> 8;
+			b = pix32;
 
 			switch (format->format) {
 			case GUD_DRM_FORMAT_XRGB1111: