直接爆破

exp:

#include <stdio.h>
#include <string.h>

int main()
{
    int enc[] =
    {
        0xE8, 0x80, 0x84, 0x08, 0x18, 0x3C, 0x78, 0x68, 0x00, 0x70, 
        0x7C, 0x94, 0xC8, 0xE0, 0x10, 0xEC, 0xB4, 0xAC, 0x68, 0xA8, 
        0x0C, 0x1C, 0x90, 0xCC, 0x54, 0x3C, 0x14, 0xDC, 0x30
    };

    char key[] = "NewStarCTF";

    for(int i =0; i < 29; i++)
    {
        for(int j = 33; j < 127; j++)
        {
            int tmp = j;
            if(tmp >= 'A' && tmp <= 'Z')
            {
                tmp = (tmp - 52) % 26 + 65;
            }
            else if(tmp >= '0' && tmp <= '9')
            {
                tmp = (tmp - 45) % 10 + 48;
            }
            else if(tmp >= 'a' && tmp <= 'z')
            {
                tmp = (tmp - 89) % 26 + 97;
            }

            tmp += key[i % strlen(key)];
            tmp = ~tmp;
            tmp = (unsigned char)(tmp * 52);

            if(tmp == enc[i])
            {
                if((j >= 'A' && j <= 'Z') || (j >= 'a' && j <= 'z'))
                {
                    printf("%c", j);
                    break;
                }
            }
        }
    }
    return 0;
}

flag:

BruteForceIsAGoodwaytoGetFlag