diff --git a/Tests/test_file_apng.py b/Tests/test_file_apng.py
index 8348da4ebca..7fb6f59d493 100644
--- a/Tests/test_file_apng.py
+++ b/Tests/test_file_apng.py
@@ -249,8 +249,8 @@ def test_apng_mode():
         assert im.mode == "P"
         im.seek(im.n_frames - 1)
         im = im.convert("RGBA")
-        assert im.getpixel((0, 0)) == (0, 255, 0, 255)
-        assert im.getpixel((64, 32)) == (0, 255, 0, 255)
+        assert im.getpixel((0, 0)) == (255, 0, 0, 0)
+        assert im.getpixel((64, 32)) == (255, 0, 0, 0)
 
     with Image.open("Tests/images/apng/mode_palette_1bit_alpha.png") as im:
         assert im.mode == "P"
diff --git a/Tests/test_file_gif.py b/Tests/test_file_gif.py
index df029dcebfb..cc2ae10c366 100644
--- a/Tests/test_file_gif.py
+++ b/Tests/test_file_gif.py
@@ -749,10 +749,10 @@ def test_rgb_transparency(tmp_path):
     # Single frame
     im = Image.new("RGB", (1, 1))
     im.info["transparency"] = (255, 0, 0)
-    pytest.warns(UserWarning, im.save, out)
+    im.save(out)
 
     with Image.open(out) as reloaded:
-        assert "transparency" not in reloaded.info
+        assert "transparency" in reloaded.info
 
     # Multiple frames
     im = Image.new("RGB", (1, 1))
diff --git a/Tests/test_image.py b/Tests/test_image.py
index 82efefc1e6c..214f2668f9a 100644
--- a/Tests/test_image.py
+++ b/Tests/test_image.py
@@ -582,6 +582,10 @@ def test_register_extensions(self):
         assert ext_individual == ext_multiple
 
     def test_remap_palette(self):
+        # Test identity transform
+        with Image.open("Tests/images/hopper.gif") as im:
+            assert_image_equal(im, im.remap_palette(list(range(256))))
+
         # Test illegal image mode
         with hopper() as im:
             with pytest.raises(ValueError):
@@ -606,7 +610,7 @@ def _make_new(base_image, im, palette_result=None):
             else:
                 assert new_im.palette is None
 
-        _make_new(im, im_p, im_p.palette)
+        _make_new(im, im_p, ImagePalette.ImagePalette(list(range(256)) * 3))
         _make_new(im_p, im, None)
         _make_new(im, blank_p, ImagePalette.ImagePalette())
         _make_new(im, blank_pa, ImagePalette.ImagePalette())
diff --git a/Tests/test_image_convert.py b/Tests/test_image_convert.py
index 6fe1bd962fd..5dcdac0e4e1 100644
--- a/Tests/test_image_convert.py
+++ b/Tests/test_image_convert.py
@@ -93,7 +93,7 @@ def test_trns_p(tmp_path):
     im_l.save(f)
 
     im_rgb = im.convert("RGB")
-    assert im_rgb.info["transparency"] == (0, 0, 0)  # undone
+    assert im_rgb.info["transparency"] == (0, 1, 2)  # undone
     im_rgb.save(f)
 
 
@@ -128,8 +128,8 @@ def test_trns_l(tmp_path):
     assert "transparency" in im_p.info
     im_p.save(f)
 
-    im_p = pytest.warns(UserWarning, im.convert, "P", palette=Image.ADAPTIVE)
-    assert "transparency" not in im_p.info
+    im_p = im.convert("P", palette=Image.ADAPTIVE)
+    assert "transparency" in im_p.info
     im_p.save(f)
 
 
@@ -155,13 +155,19 @@ def test_trns_RGB(tmp_path):
     assert "transparency" not in im_p.info
     im_p.save(f)
 
+    im = Image.new("RGB", (1, 1))
+    im.info["transparency"] = im.getpixel((0, 0))
+    im_p = im.convert("P", palette=Image.ADAPTIVE)
+    assert im_p.info["transparency"] == im_p.getpixel((0, 0))
+    im_p.save(f)
+
 
 def test_gif_with_rgba_palette_to_p():
     # See https://github.com/python-pillow/Pillow/issues/2433
     with Image.open("Tests/images/hopper.gif") as im:
         im.info["transparency"] = 255
         im.load()
-        assert im.palette.mode == "RGBA"
+        assert im.palette.mode == "RGB"
         im_p = im.convert("P")
 
     # Should not raise ValueError: unrecognized raw mode
diff --git a/Tests/test_imageops.py b/Tests/test_imageops.py
index 31725581f35..acd20bb5860 100644
--- a/Tests/test_imageops.py
+++ b/Tests/test_imageops.py
@@ -157,9 +157,16 @@ def test_scale():
 
 
 def test_expand_palette():
-    im = Image.open("Tests/images/hopper.gif")
-    im_expanded = ImageOps.expand(im)
-    assert_image_equal(im_expanded.convert("RGB"), im.convert("RGB"))
+    im = Image.open("Tests/images/p_16.tga")
+    im_expanded = ImageOps.expand(im, 10, (255, 0, 0))
+
+    px = im_expanded.convert("RGB").load()
+    assert px[0, 0] == (255, 0, 0)
+
+    im_cropped = im_expanded.crop(
+        (10, 10, im_expanded.width - 10, im_expanded.height - 10)
+    )
+    assert_image_equal(im_cropped, im)
 
 
 def test_colorize_2color():
diff --git a/Tests/test_imagepalette.py b/Tests/test_imagepalette.py
index 0ea2472a989..ee2cfdf9b2b 100644
--- a/Tests/test_imagepalette.py
+++ b/Tests/test_imagepalette.py
@@ -2,27 +2,47 @@
 
 from PIL import Image, ImagePalette
 
-from .helper import assert_image_equal_tofile
+from .helper import assert_image_equal, assert_image_equal_tofile
 
 
 def test_sanity():
 
-    ImagePalette.ImagePalette("RGB", list(range(256)) * 3)
+    palette = ImagePalette.ImagePalette("RGB", list(range(256)) * 3)
+    assert len(palette.colors) == 256
+
     with pytest.raises(ValueError):
-        ImagePalette.ImagePalette("RGB", list(range(256)) * 2)
+        ImagePalette.ImagePalette("RGB", list(range(256)) * 3, 10)
+
+
+def test_reload():
+    im = Image.open("Tests/images/hopper.gif")
+    original = im.copy()
+    im.palette.dirty = 1
+    assert_image_equal(im.convert("RGB"), original.convert("RGB"))
 
 
 def test_getcolor():
 
     palette = ImagePalette.ImagePalette()
+    assert len(palette.palette) == 0
+    assert len(palette.colors) == 0
 
     test_map = {}
     for i in range(256):
         test_map[palette.getcolor((i, i, i))] = i
-
     assert len(test_map) == 256
+
+    # Colors can be converted between RGB and RGBA
+    rgba_palette = ImagePalette.ImagePalette("RGBA")
+    assert rgba_palette.getcolor((0, 0, 0)) == rgba_palette.getcolor((0, 0, 0, 255))
+
+    assert palette.getcolor((0, 0, 0)) == palette.getcolor((0, 0, 0, 255))
+
+    # An error is raised when the palette is full
     with pytest.raises(ValueError):
         palette.getcolor((1, 2, 3))
+    # But not if the image is not using one of the palette entries
+    palette.getcolor((1, 2, 3), image=Image.new("P", (1, 1)))
 
     # Test unknown color specifier
     with pytest.raises(ValueError):
@@ -116,7 +136,7 @@ def test_getdata():
     mode, data_out = palette.getdata()
 
     # Assert
-    assert mode == "RGB;L"
+    assert mode == "RGB"
 
 
 def test_rawmode_getdata():
diff --git a/src/PIL/GifImagePlugin.py b/src/PIL/GifImagePlugin.py
index 5c93de2c9be..bb88761e68f 100644
--- a/src/PIL/GifImagePlugin.py
+++ b/src/PIL/GifImagePlugin.py
@@ -461,10 +461,10 @@ def _write_multiple_frames(im, fp, palette):
                 previous = im_frames[-1]
                 if encoderinfo.get("disposal") == 2:
                     if background_im is None:
-                        background = _get_background(
-                            im,
-                            im.encoderinfo.get("background", im.info.get("background")),
+                        color = im.encoderinfo.get(
+                            "transparency", im.info.get("transparency", (0, 0, 0))
                         )
+                        background = _get_background(im_frame, color)
                         background_im = Image.new("P", im_frame.size, background)
                         background_im.putpalette(im_frames[0]["im"].palette)
                     base_im = background_im
@@ -760,7 +760,15 @@ def _get_background(im, infoBackground):
             # WebPImagePlugin stores an RGBA value in info["background"]
             # So it must be converted to the same format as GifImagePlugin's
             # info["background"] - a global color table index
-            background = im.palette.getcolor(background)
+            try:
+                background = im.palette.getcolor(background, im)
+            except ValueError as e:
+                if str(e) == "cannot allocate more than 256 colors":
+                    # If all 256 colors are in use,
+                    # then there is no need for the background color
+                    return 0
+                else:
+                    raise
     return background
 
 
diff --git a/src/PIL/Image.py b/src/PIL/Image.py
index 8fb7ea57698..3ba38285bc4 100644
--- a/src/PIL/Image.py
+++ b/src/PIL/Image.py
@@ -833,7 +833,7 @@ def load(self):
             palette_length = self.im.putpalette(mode, arr)
             self.palette.dirty = 0
             self.palette.rawmode = None
-            if "transparency" in self.info:
+            if "transparency" in self.info and mode in ("RGBA", "LA", "PA"):
                 if isinstance(self.info["transparency"], int):
                     self.im.putpalettealpha(self.info["transparency"], 0)
                 else:
@@ -977,21 +977,28 @@ def convert_transparency(m, v):
                     if self.mode == "P":
                         trns_im.putpalette(self.palette)
                         if isinstance(t, tuple):
+                            err = "Couldn't allocate a palette color for transparency"
                             try:
-                                t = trns_im.palette.getcolor(t)
-                            except Exception as e:
-                                raise ValueError(
-                                    "Couldn't allocate a palette color for transparency"
-                                ) from e
-                    trns_im.putpixel((0, 0), t)
-
-                    if mode in ("L", "RGB"):
-                        trns_im = trns_im.convert(mode)
+                                t = trns_im.palette.getcolor(t, self)
+                            except ValueError as e:
+                                if str(e) == "cannot allocate more than 256 colors":
+                                    # If all 256 colors are in use,
+                                    # then there is no need for transparency
+                                    t = None
+                                else:
+                                    raise ValueError(err) from e
+                    if t is None:
+                        trns = None
                     else:
-                        # can't just retrieve the palette number, got to do it
-                        # after quantization.
-                        trns_im = trns_im.convert("RGB")
-                    trns = trns_im.getpixel((0, 0))
+                        trns_im.putpixel((0, 0), t)
+
+                        if mode in ("L", "RGB"):
+                            trns_im = trns_im.convert(mode)
+                        else:
+                            # can't just retrieve the palette number, got to do it
+                            # after quantization.
+                            trns_im = trns_im.convert("RGB")
+                        trns = trns_im.getpixel((0, 0))
 
             elif self.mode == "P" and mode == "RGBA":
                 t = self.info["transparency"]
@@ -1009,14 +1016,14 @@ def convert_transparency(m, v):
             new = self._new(im)
             from . import ImagePalette
 
-            new.palette = ImagePalette.raw("RGB", new.im.getpalette("RGB"))
+            new.palette = ImagePalette.ImagePalette("RGB", new.im.getpalette("RGB"))
             if delete_trns:
                 # This could possibly happen if we requantize to fewer colors.
                 # The transparency would be totally off in that case.
                 del new.info["transparency"]
             if trns is not None:
                 try:
-                    new.info["transparency"] = new.palette.getcolor(trns)
+                    new.info["transparency"] = new.palette.getcolor(trns, new)
                 except Exception:
                     # if we can't make a transparent color, don't leave the old
                     # transparency hanging around to mess us up.
@@ -1039,16 +1046,25 @@ def convert_transparency(m, v):
                 raise ValueError("illegal conversion") from e
 
         new_im = self._new(im)
+        if mode == "P" and palette != ADAPTIVE:
+            from . import ImagePalette
+
+            new_im.palette = ImagePalette.ImagePalette("RGB", list(range(256)) * 3)
         if delete_trns:
             # crash fail if we leave a bytes transparency in an rgb/l mode.
             del new_im.info["transparency"]
         if trns is not None:
             if new_im.mode == "P":
                 try:
-                    new_im.info["transparency"] = new_im.palette.getcolor(trns)
-                except Exception:
+                    new_im.info["transparency"] = new_im.palette.getcolor(trns, new_im)
+                except ValueError as e:
                     del new_im.info["transparency"]
-                    warnings.warn("Couldn't allocate palette entry for transparency")
+                    if str(e) != "cannot allocate more than 256 colors":
+                        # If all 256 colors are in use,
+                        # then there is no need for transparency
+                        warnings.warn(
+                            "Couldn't allocate palette entry for transparency"
+                        )
             else:
                 new_im.info["transparency"] = trns
         return new_im
@@ -1700,7 +1716,7 @@ def putpalette(self, data, rawmode="RGB"):
         Attaches a palette to this image.  The image must be a "P", "PA", "L"
         or "LA" image.
 
-        The palette sequence must contain either 768 integer values, or 1024
+        The palette sequence must contain at most 768 integer values, or 1024
         integer values if alpha is included. Each group of values represents
         the red, green, blue (and alpha if included) values for the
         corresponding pixel index. Instead of an integer sequence, you can use
@@ -1713,7 +1729,6 @@ def putpalette(self, data, rawmode="RGB"):
 
         if self.mode not in ("L", "LA", "P", "PA"):
             raise ValueError("illegal image mode")
-        self.load()
         if isinstance(data, ImagePalette.ImagePalette):
             palette = ImagePalette.raw(data.rawmode, data.palette)
         else:
@@ -1760,7 +1775,7 @@ def putpixel(self, xy, value):
             and len(value) in [3, 4]
         ):
             # RGB or RGBA value for a P image
-            value = self.palette.getcolor(value)
+            value = self.palette.getcolor(value, self)
         return self.im.putpixel(xy, value)
 
     def remap_palette(self, dest_map, source_palette=None):
@@ -1781,6 +1796,7 @@ def remap_palette(self, dest_map, source_palette=None):
 
         if source_palette is None:
             if self.mode == "P":
+                self.load()
                 real_source_palette = self.im.getpalette("RGB")[:768]
             else:  # L-mode
                 real_source_palette = bytearray(i // 3 for i in range(768))
@@ -1818,23 +1834,19 @@ def remap_palette(self, dest_map, source_palette=None):
         m_im = self.copy()
         m_im.mode = "P"
 
-        m_im.palette = ImagePalette.ImagePalette(
-            "RGB", palette=mapping_palette * 3, size=768
-        )
+        m_im.palette = ImagePalette.ImagePalette("RGB", palette=mapping_palette * 3)
         # possibly set palette dirty, then
         # m_im.putpalette(mapping_palette, 'L')  # converts to 'P'
         # or just force it.
         # UNDONE -- this is part of the general issue with palettes
-        m_im.im.putpalette(*m_im.palette.getdata())
+        m_im.im.putpalette("RGB;L", m_im.palette.tobytes())
 
         m_im = m_im.convert("L")
 
         # Internally, we require 768 bytes for a palette.
         new_palette_bytes = palette_bytes + (768 - len(palette_bytes)) * b"\x00"
         m_im.putpalette(new_palette_bytes)
-        m_im.palette = ImagePalette.ImagePalette(
-            "RGB", palette=palette_bytes, size=len(palette_bytes)
-        )
+        m_im.palette = ImagePalette.ImagePalette("RGB", palette=palette_bytes)
 
         return m_im
 
diff --git a/src/PIL/ImageDraw.py b/src/PIL/ImageDraw.py
index 717fb48a48e..aea0cc680a8 100644
--- a/src/PIL/ImageDraw.py
+++ b/src/PIL/ImageDraw.py
@@ -70,6 +70,7 @@ def __init__(self, im, mode=None):
             self.palette = im.palette
         else:
             self.palette = None
+        self._image = im
         self.im = im.im
         self.draw = Image.core.draw(self.im, blend)
         self.mode = mode
@@ -108,13 +109,13 @@ def _getink(self, ink, fill=None):
                 if isinstance(ink, str):
                     ink = ImageColor.getcolor(ink, self.mode)
                 if self.palette and not isinstance(ink, numbers.Number):
-                    ink = self.palette.getcolor(ink)
+                    ink = self.palette.getcolor(ink, self._image)
                 ink = self.draw.draw_ink(ink)
             if fill is not None:
                 if isinstance(fill, str):
                     fill = ImageColor.getcolor(fill, self.mode)
                 if self.palette and not isinstance(fill, numbers.Number):
-                    fill = self.palette.getcolor(fill)
+                    fill = self.palette.getcolor(fill, self._image)
                 fill = self.draw.draw_ink(fill)
         return ink, fill
 
diff --git a/src/PIL/ImageOps.py b/src/PIL/ImageOps.py
index ad07fcedb5c..98be4bb919a 100644
--- a/src/PIL/ImageOps.py
+++ b/src/PIL/ImageOps.py
@@ -20,7 +20,7 @@
 import functools
 import operator
 
-from . import Image
+from . import Image, ImageDraw
 
 #
 # helpers
@@ -392,10 +392,17 @@ def expand(image, border=0, fill=0):
     left, top, right, bottom = _border(border)
     width = left + image.size[0] + right
     height = top + image.size[1] + bottom
-    out = Image.new(image.mode, (width, height), _color(fill, image.mode))
+    color = _color(fill, image.mode)
     if image.mode == "P" and image.palette:
+        out = Image.new(image.mode, (width, height))
         out.putpalette(image.palette)
-    out.paste(image, (left, top))
+        out.paste(image, (left, top))
+
+        draw = ImageDraw.Draw(out)
+        draw.rectangle((0, 0, width, height), outline=color, width=border)
+    else:
+        out = Image.new(image.mode, (width, height), color)
+        out.paste(image, (left, top))
     return out
 
 
diff --git a/src/PIL/ImagePalette.py b/src/PIL/ImagePalette.py
index d0604112fd9..76c4c46d551 100644
--- a/src/PIL/ImagePalette.py
+++ b/src/PIL/ImagePalette.py
@@ -39,14 +39,27 @@ class ImagePalette:
     def __init__(self, mode="RGB", palette=None, size=0):
         self.mode = mode
         self.rawmode = None  # if set, palette contains raw data
-        self.palette = palette or bytearray(range(256)) * len(self.mode)
-        self.colors = {}
+        self.palette = palette or bytearray()
         self.dirty = None
-        if (size == 0 and len(self.mode) * 256 != len(self.palette)) or (
-            size != 0 and size != len(self.palette)
-        ):
+        if size != 0 and size != len(self.palette):
             raise ValueError("wrong palette size")
 
+    @property
+    def palette(self):
+        return self._palette
+
+    @palette.setter
+    def palette(self, palette):
+        self._palette = palette
+
+        mode_len = len(self.mode)
+        self.colors = {}
+        for i in range(0, len(self.palette), mode_len):
+            color = tuple(self.palette[i : i + mode_len])
+            if color in self.colors:
+                continue
+            self.colors[color] = i // mode_len
+
     def copy(self):
         new = ImagePalette()
 
@@ -54,7 +67,6 @@ def copy(self):
         new.rawmode = self.rawmode
         if self.palette is not None:
             new.palette = self.palette[:]
-        new.colors = self.colors.copy()
         new.dirty = self.dirty
 
         return new
@@ -68,7 +80,7 @@ def getdata(self):
         """
         if self.rawmode:
             return self.rawmode, self.palette
-        return self.mode + ";L", self.tobytes()
+        return self.mode, self.tobytes()
 
     def tobytes(self):
         """Convert palette to bytes.
@@ -80,14 +92,12 @@ def tobytes(self):
         if isinstance(self.palette, bytes):
             return self.palette
         arr = array.array("B", self.palette)
-        if hasattr(arr, "tobytes"):
-            return arr.tobytes()
-        return arr.tostring()
+        return arr.tobytes()
 
     # Declare tostring as an alias for tobytes
     tostring = tobytes
 
-    def getcolor(self, color):
+    def getcolor(self, color, image=None):
         """Given an rgb tuple, allocate palette entry.
 
         .. warning:: This method is experimental.
@@ -95,19 +105,37 @@ def getcolor(self, color):
         if self.rawmode:
             raise ValueError("palette contains raw palette data")
         if isinstance(color, tuple):
+            if self.mode == "RGB":
+                if len(color) == 4 and color[3] == 255:
+                    color = color[:3]
+            elif self.mode == "RGBA":
+                if len(color) == 3:
+                    color += (255,)
             try:
                 return self.colors[color]
             except KeyError as e:
                 # allocate new color slot
-                if isinstance(self.palette, bytes):
-                    self.palette = bytearray(self.palette)
-                index = len(self.colors)
+                if not isinstance(self.palette, bytearray):
+                    self._palette = bytearray(self.palette)
+                index = len(self.palette) // 3
                 if index >= 256:
-                    raise ValueError("cannot allocate more than 256 colors") from e
+                    if image:
+                        # Search for an unused index
+                        for i, count in reversed(list(enumerate(image.histogram()))):
+                            if count == 0:
+                                index = i
+                                break
+                    if index >= 256:
+                        raise ValueError("cannot allocate more than 256 colors") from e
                 self.colors[color] = index
-                self.palette[index] = color[0]
-                self.palette[index + 256] = color[1]
-                self.palette[index + 512] = color[2]
+                if index * 3 < len(self.palette):
+                    self._palette = (
+                        self.palette[: index * 3]
+                        + bytes(color)
+                        + self.palette[index * 3 + 3 :]
+                    )
+                else:
+                    self._palette += bytes(color)
                 self.dirty = 1
                 return index
         else:
diff --git a/src/PIL/PyAccess.py b/src/PIL/PyAccess.py
index 494f5f9f478..5ceaa238a8f 100644
--- a/src/PIL/PyAccess.py
+++ b/src/PIL/PyAccess.py
@@ -54,6 +54,7 @@ def __init__(self, img, readonly=False):
         self.image32 = ffi.cast("int **", vals["image32"])
         self.image = ffi.cast("unsigned char **", vals["image"])
         self.xsize, self.ysize = img.im.size
+        self._img = img
 
         # Keep pointer to im object to prevent dereferencing.
         self._im = img.im
@@ -93,7 +94,7 @@ def __setitem__(self, xy, color):
             and len(color) in [3, 4]
         ):
             # RGB or RGBA value for a P image
-            color = self._palette.getcolor(color)
+            color = self._palette.getcolor(color, self._img)
 
         return self.set_pixel(x, y, color)
 
diff --git a/src/PIL/WebPImagePlugin.py b/src/PIL/WebPImagePlugin.py
index 50b82feb048..b63a07ca8e3 100644
--- a/src/PIL/WebPImagePlugin.py
+++ b/src/PIL/WebPImagePlugin.py
@@ -320,7 +320,7 @@ def _save(im, fp, filename):
         alpha = (
             "A" in im.mode
             or "a" in im.mode
-            or (im.mode == "P" and "A" in im.im.getpalettemode())
+            or (im.mode == "P" and "transparency" in im.info)
         )
         im = im.convert("RGBA" if alpha else "RGB")