Encoding functions in C/C++

BYTE* base64enc(BYTE* in, INT inlength)
{
	BYTE alphabet[] = {
				'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
				'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
				'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X',
				'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
				'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
				'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
				'w', 'x', 'y', 'z', '0', '1', '2', '3',
				'4', '5', '6', '7', '8', '9', '+', '/'
	};

	BYTE mask0 = 0b11111100;
	BYTE mask1 = 0b00000011;
	BYTE mask2 = 0b11110000;
	BYTE mask3 = 0b00001111;
	BYTE mask4 = 0b11000000;
	BYTE mask5 = 0b00111111;

	INT iter = inlength + (3 - (inlength % 3));
	INT lenbuffer = (iter / 3) * 4;

	BYTE* b64encode = (BYTE*)malloc(lenbuffer);

	INT i = lenbuffer;
	if (inlength % 3 != 0)
	{
		b64encode[i - 1] = '=';
		if (inlength % 3 == 1)
		{
			b64encode[i - 2] = '=';
		}
		else if (inlength % 3 == 2)
		{
			b64encode[i - 2] = alphabet[((mask3 & in[iter - 2]) << 2) ^ ((mask4 & in[iter - 1]) >> 6)];
		}
		b64encode[i - 3] = alphabet[((mask1 & in[iter - 3]) << 4) ^ ((mask2 & in[iter - 2]) >> 4)];
		b64encode[i - 4] = alphabet[(mask0 & in[iter - 3]) >> 2];
		i = lenbuffer - 4;
		iter = iter - 3;
	}

	while (i > 0) {
		b64encode[i - 4] = alphabet[(mask0 & in[iter - 3]) >> 2];
		b64encode[i - 3] = alphabet[((mask1 & in[iter - 3]) << 4) ^ ((mask2 & in[iter - 2]) >> 4)];
		b64encode[i - 2] = alphabet[((mask3 & in[iter - 2]) << 2) ^ ((mask4 & in[iter - 1]) >> 6)];
		b64encode[i - 1] = alphabet[mask5 & in[iter - 1]];
		i = i - 4;
		iter = iter - 3;
	}

	return b64encode;
}