diff --git a/Common/Data/Convert/ColorConv.h b/Common/Data/Convert/ColorConv.h
index 53a286d1273a..6c47702b7025 100644
--- a/Common/Data/Convert/ColorConv.h
+++ b/Common/Data/Convert/ColorConv.h
@@ -64,26 +64,28 @@ inline u32 RGBA4444ToRGBA8888(u16 src) {
 	const u32 g = (src & 0x00F0) << 4;
 	const u32 b = (src & 0x0F00) << 8;
 	const u32 a = (src & 0xF000) << 12;
-
 	const u32 c = r | g | b | a;
 	return c | (c << 4);
 }
 
 inline u32 RGBA5551ToRGBA8888(u16 src) {
-	u8 r = Convert5To8((src >> 0) & 0x1F);
-	u8 g = Convert5To8((src >> 5) & 0x1F);
-	u8 b = Convert5To8((src >> 10) & 0x1F);
-	u8 a = (src >> 15) & 0x1;
-	a = (a) ? 0xff : 0;
-	return (a << 24) | (b << 16) | (g << 8) | r;
+	u32 dark = ((src & 0x1F) << 3) | ((src & 0x3E0) << 6) | ((src & 0x7C00) << 9);
+	// Replicate the top 3 upper bits into the missing lower bits.
+	u32 full = (dark | ((dark >> 5) & 0x070707));
+	if (src >> 15) {
+		full |= 0xFF000000;
+	}
+	return full;
 }
 
 inline u32 RGB565ToRGBA8888(u16 src) {
-	u8 r = Convert5To8((src >> 0) & 0x1F);
-	u8 g = Convert6To8((src >> 5) & 0x3F);
-	u8 b = Convert5To8((src >> 11) & 0x1F);
-	u8 a = 0xFF;
-	return (a << 24) | (b << 16) | (g << 8) | r;
+	u32 dark_rb = ((src & 0x1F) << 3) | ((src & 0xF800) << 8);
+	// Replicate the top 3 upper bits into the missing lower bits.
+	u32 full_rb = (dark_rb | ((dark_rb >> 5) & 0x070007));
+	// Add in green (6 bits instead of 5).
+	u32 dark_g = ((src & 0x7E0) << 5);
+	u32 full_g = dark_g | ((dark_g >> 6) & 0x300);
+	return full_rb | full_g | 0xFF000000;
 }
 
 inline u16 RGBA8888ToRGB565(u32 value) {
diff --git a/GPU/Common/VertexDecoderCommon.cpp b/GPU/Common/VertexDecoderCommon.cpp
index 243300e457ed..85d57aa4b8b0 100644
--- a/GPU/Common/VertexDecoderCommon.cpp
+++ b/GPU/Common/VertexDecoderCommon.cpp
@@ -506,40 +506,39 @@ void VertexDecoder::Step_ColorInvalid() const
 
 void VertexDecoder::Step_Color565() const
 {
-	u8 *c = decoded_ + decFmt.c0off;
 	u16 cdata = *(const u16_le *)(ptr_ + coloff);
-	c[0] = Convert5To8(cdata & 0x1f);
-	c[1] = Convert6To8((cdata >> 5) & 0x3f);
-	c[2] = Convert5To8((cdata >> 11) & 0x1f);
-	c[3] = 255;
-	// Always full alpha.
+	u32 *c = (u32 *)(decoded_ + decFmt.c0off);
+	*c = RGB565ToRGBA8888(cdata);
 }
 
 void VertexDecoder::Step_Color5551() const
 {
-	u8 *c = decoded_ + decFmt.c0off;
 	u16 cdata = *(const u16_le *)(ptr_ + coloff);
-	gstate_c.vertexFullAlpha = gstate_c.vertexFullAlpha && (cdata >> 15) != 0;
-	c[0] = Convert5To8(cdata & 0x1f);
-	c[1] = Convert5To8((cdata >> 5) & 0x1f);
-	c[2] = Convert5To8((cdata >> 10) & 0x1f);
-	c[3] = (cdata >> 15) ? 255 : 0;
+	u32 *c = (u32 *)(decoded_ + decFmt.c0off);
+	int alpha = (cdata >> 15);
+	if (!alpha) {
+		gstate_c.vertexFullAlpha = false;
+	}
+	*c = RGBA5551ToRGBA8888(cdata);
 }
 
 void VertexDecoder::Step_Color4444() const
 {
-	u8 *c = decoded_ + decFmt.c0off;
 	u16 cdata = *(const u16_le *)(ptr_ + coloff);
-	gstate_c.vertexFullAlpha = gstate_c.vertexFullAlpha && (cdata >> 12) == 0xF;
-	for (int j = 0; j < 4; j++)
-		c[j] = Convert4To8((cdata >> (j * 4)) & 0xF);
+	u32 *c = (u32 *)(decoded_ + decFmt.c0off);
+	if ((cdata >> 12) != 0xF) {
+		gstate_c.vertexFullAlpha = false;
+	}
+	*c = RGBA4444ToRGBA8888(cdata);
 }
 
 void VertexDecoder::Step_Color8888() const
 {
 	u8 *c = decoded_ + decFmt.c0off;
 	const u8 *cdata = (const u8*)(ptr_ + coloff);
-	gstate_c.vertexFullAlpha = gstate_c.vertexFullAlpha && cdata[3] == 255;
+	if (cdata[3] != 255) {
+		gstate_c.vertexFullAlpha = false;
+	}
 	memcpy(c, cdata, sizeof(u8) * 4);
 }
 
diff --git a/unittest/UnitTest.cpp b/unittest/UnitTest.cpp
index e482b6f0e296..640e8eddb9d6 100644
--- a/unittest/UnitTest.cpp
+++ b/unittest/UnitTest.cpp
@@ -67,6 +67,7 @@
 #include "Common/Log.h"
 #include "Common/StringUtils.h"
 #include "Core/Config.h"
+#include "Common/Data/Convert/ColorConv.h"
 #include "Common/File/VFS/VFS.h"
 #include "Common/File/VFS/DirectoryReader.h"
 #include "Core/FileSystems/ISOFileSystem.h"
@@ -997,6 +998,40 @@ bool TestIniFile() {
 	return true;
 }
 
+inline u32 ReferenceRGBA5551ToRGBA8888(u16 src) {
+	u8 r = Convert5To8((src >> 0) & 0x1F);
+	u8 g = Convert5To8((src >> 5) & 0x1F);
+	u8 b = Convert5To8((src >> 10) & 0x1F);
+	u8 a = (src >> 15) & 0x1;
+	a = (a) ? 0xff : 0;
+	return (a << 24) | (b << 16) | (g << 8) | r;
+}
+
+inline u32 ReferenceRGB565ToRGBA8888(u16 src) {
+	u8 r = Convert5To8((src >> 0) & 0x1F);
+	u8 g = Convert6To8((src >> 5) & 0x3F);
+	u8 b = Convert5To8((src >> 11) & 0x1F);
+	u8 a = 0xFF;
+	return (a << 24) | (b << 16) | (g << 8) | r;
+}
+
+bool TestColorConv() {
+	// Can exhaustively test the 16->32 conversions.
+	for (int i = 0; i < 65536; i++) {
+		u16 col16 = i;
+
+		u32 reference = ReferenceRGBA5551ToRGBA8888(col16);
+		u32 value = RGBA5551ToRGBA8888(col16);
+		EXPECT_EQ_INT(reference, value);
+
+		reference = ReferenceRGB565ToRGBA8888(col16);
+		value = RGB565ToRGBA8888(col16);
+		EXPECT_EQ_INT(reference, value);
+	}
+
+	return true;
+}
+
 typedef bool (*TestFunc)();
 struct TestItem {
 	const char *name;
@@ -1056,6 +1091,7 @@ TestItem availableTests[] = {
 	TEST_ITEM(VFS),
 	TEST_ITEM(Substitutions),
 	TEST_ITEM(IniFile),
+	TEST_ITEM(ColorConv),
 };
 
 int main(int argc, const char *argv[]) {