sixel: fix hls-to-rgb and color count issues (#185)

This fixes the following sixel issues:

- The HLS to RGB color conversion is broken.
  Ref. d4ade1fe3c

- Not enough color registers have been allocated for 1024 colors.
  Ref. 97f93e8436

sixel.c

@@ -100,9 +100,13 @@ set_default_color(sixel_image_t *image)
 		image->palette[n++] = SIXEL_RGB(i * 11, i * 11, i * 11);
 	}
 
+	/* sixels rarely use more than 256 colors and if they do, they use a custom
+	 * palette, so we don't need to initialize these colors */
+	/*
 	for (; n < DECSIXEL_PALETTE_MAX; n++) {
 		image->palette[n] = SIXEL_RGB(255, 255, 255);
 	}
+	*/
 
 	return (0);
 }
@@ -610,11 +614,12 @@ sixel_parser_parse(sixel_state_t *st, const unsigned char *p, size_t len)
 				st->param = 0;
 
 				if (st->nparams > 0) {
-					st->color_index = 1 + st->params[0];  /* offset 1(background color) added */
+					st->color_index = st->params[0];
 					if (st->color_index < 0)
 						st->color_index = 0;
 					else if (st->color_index >= DECSIXEL_PALETTE_MAX)
 						st->color_index = DECSIXEL_PALETTE_MAX - 1;
+					st->color_index++; /* offset by 1 (background) */
 				}
 
 				if (st->nparams > 4) {

sixel.h

@@ -14,7 +14,7 @@ typedef struct sixel_image_buffer {
 	sixel_color_no_t *data;
 	int width;
 	int height;
-	sixel_color_t palette[DECSIXEL_PALETTE_MAX];
+	sixel_color_t palette[DECSIXEL_PALETTE_MAX + 1];
 	sixel_color_no_t ncolors;
 	int palette_modified;
 	int use_private_register;

sixel_hls.c

@@ -30,20 +30,18 @@
 // sale, use or other dealings in this Software without prior written
 // authorization.
 
-#define SIXEL_RGB(r, g, b) ((r) + ((g) << 8) + ((b) << 16) + (255 << 24))
+#define SIXEL_RGB(r, g, b) ((255 << 24) + ((r) << 16) + ((g) << 8) +  (b))
 
 int
 hls_to_rgb(int hue, int lum, int sat)
 {
-  double hs = (hue + 240) % 360;
-  double hv = hs / 360.0;
   double lv = lum / 100.0;
   double sv = sat / 100.0;
   double c, x, m, c2;
   double r1, g1, b1;
-  int r, g, b;
+  int r, g, b, hs;
-  int hpi;
 
+  hue = (hue + 240) % 360;
   if (sat == 0) {
     r = g = b = lum * 255 / 100;
     return SIXEL_RGB(r, g, b);
@@ -52,12 +50,14 @@ hls_to_rgb(int hue, int lum, int sat)
   if ((c2 = ((2.0 * lv) - 1.0)) < 0.0) {
     c2 = -c2;
   }
+  if ((hs = (hue % 120) - 60) < 0) {
+    hs = -hs;
+  }
   c = (1.0 - c2) * sv;
-  hpi = (int) (hv * 6.0);
+  x = ((60 - hs) / 60.0) * c;
-  x = (hpi & 1) ? c : 0.0;
   m = lv - 0.5 * c;
 
-  switch (hpi) {
+  switch (hue / 60) {
   case 0:
     r1 = c;
     g1 = x;
@@ -92,24 +92,24 @@ hls_to_rgb(int hue, int lum, int sat)
     return SIXEL_RGB(255, 255, 255);
   }
 
-  r = (int) ((r1 + m) * 100.0 + 0.5);
+  r = (int) ((r1 + m) * 255.0 + 0.5);
-  g = (int) ((g1 + m) * 100.0 + 0.5);
+  g = (int) ((g1 + m) * 255.0 + 0.5);
-  b = (int) ((b1 + m) * 100.0 + 0.5);
+  b = (int) ((b1 + m) * 255.0 + 0.5);
 
   if (r < 0) {
     r = 0;
-  } else if (r > 100) {
+  } else if (r > 255) {
-    r = 100;
+    r = 255;
   }
   if (g < 0) {
     g = 0;
-  } else if (g > 100) {
+  } else if (g > 255) {
-    g = 100;
+    g = 255;
   }
   if (b < 0) {
     b = 0;
-  } else if (b > 100) {
+  } else if (b > 255) {
-    b = 100;
+    b = 255;
   }
-  return SIXEL_RGB(r * 255 / 100, g * 255 / 100, b * 255 / 100);
+  return SIXEL_RGB(r, g, b);
 }