vm-me

Solved by: grb

This is what the challenge looks like, now there's 2 binary file attached with it. One of them is called "vm.me" and the other one is "vmme.bin"

Let's try to open up the "vm.me" binary under IDA, as usual. This is the function that will greet us.

Here, we can see that the program reads the file from the path that the user supplied through the argument, and pass the data from the file to run function. Below is the decompilation of the run function, cleaned up.

int __fastcall run(__int64 pInstructionBuf, unsigned __int64 InstructionSize)
{
  int result; // eax
  unsigned __int64 v4; // rax
  unsigned __int64 v5; // rax
  unsigned __int64 v6; // rax
  unsigned __int64 v7; // rax
  unsigned __int64 v8; // rax
  unsigned __int64 v9; // rax
  unsigned __int64 v10; // rax
  unsigned __int8 regIndex1; // [rsp+1Ch] [rbp-14h]
  unsigned __int8 regIndex2; // [rsp+1Eh] [rbp-12h]
  unsigned __int8 regIndex3; // [rsp+20h] [rbp-10h]
  unsigned __int8 v14; // [rsp+22h] [rbp-Eh]
  unsigned __int8 v15; // [rsp+26h] [rbp-Ah]
  unsigned __int8 CurrentInstruction; // [rsp+27h] [rbp-9h]
  unsigned __int64 ProgramCounter; // [rsp+28h] [rbp-8h] MAPDST
  unsigned __int64 NextProgramCounter; // [rsp+28h] [rbp-8h]

  ProgramCounter = 0;
  while ( 2 )
  {
    result = ProgramCounter;
    if ( ProgramCounter < InstructionSize )
    {
      NextProgramCounter = ProgramCounter + 1;
      CurrentInstruction = *(_BYTE *)(pInstructionBuf + ProgramCounter);
      result = CurrentInstruction;
      if ( CurrentInstruction > 8u )
      {
        if ( CurrentInstruction == 255 )
          return result;
      }
      else if ( CurrentInstruction )
      {
        switch ( CurrentInstruction )
        {
          case 1u:                              // assign
            regIndex1 = *(_BYTE *)(pInstructionBuf + NextProgramCounter);
            v4 = NextProgramCounter + 1;
            ProgramCounter = NextProgramCounter + 2;
            regs[regIndex1] = *(_BYTE *)(pInstructionBuf + v4);
            continue;
          case 2u:                              // add
            regIndex2 = *(_BYTE *)(pInstructionBuf + NextProgramCounter);
            v5 = NextProgramCounter + 1;
            ProgramCounter = NextProgramCounter + 2;
            regs[regIndex2] += regs[*(unsigned __int8 *)(pInstructionBuf + v5)];
            continue;
          case 3u:                              // XOR
            regIndex3 = *(_BYTE *)(pInstructionBuf + NextProgramCounter);
            v6 = NextProgramCounter + 1;
            ProgramCounter = NextProgramCounter + 2;
            regs[regIndex3] ^= regs[*(unsigned __int8 *)(pInstructionBuf + v6)];
            continue;
          case 4u:                              // cmp
            v14 = *(_BYTE *)(pInstructionBuf + NextProgramCounter);
            v7 = NextProgramCounter + 1;
            ProgramCounter = NextProgramCounter + 2;
            flag_zero = *(_BYTE *)(pInstructionBuf + v7) == regs[v14];
            continue;
          case 5u:                              // set pc
            ProgramCounter = *(unsigned __int8 *)(pInstructionBuf + NextProgramCounter);
            continue;
          case 6u:
            v8 = NextProgramCounter;
            ProgramCounter = NextProgramCounter + 1;
            if ( !flag_zero )
              ProgramCounter = *(unsigned __int8 *)(pInstructionBuf + v8);
            continue;
          case 7u:                              // print
            v9 = NextProgramCounter;
            ProgramCounter = NextProgramCounter + 1;
            putchar(regs[*(unsigned __int8 *)(pInstructionBuf + v9)]);
            continue;
          case 8u:                              // input
            v10 = NextProgramCounter;
            ProgramCounter = NextProgramCounter + 1;
            v15 = *(_BYTE *)(pInstructionBuf + v10);
            regs[v15] = getchar();
            continue;
          default:
            return printf("Unknown opcode: %02x\n", CurrentInstruction);
        }
      }
      return printf("Unknown opcode: %02x\n", CurrentInstruction);
    }
    break;
  }
  return result;
}

This is what we call a virtual machine-based packer. A normal assembly instruction converted into a custom instruction set that only the virtual machine understands. The function above is the virtual machine instruction decoder. Virtual machine-based packer is already common in the wild, commercial packers like VMProtect is one of them. These kind of packers are so good, a lot of game development and anti-cheat companies use them to protect their product from reverse engineers.

From the instruction decoder code above, we can map each instruction like this

• 0x01 = assign: [01][reg][imm] -> regs[reg] = imm
• 0x02 = add: [02][reg][reg2] -> regs[reg] += regs[reg2] (8-bit wrap)
• 0x03 = xor: [03][reg][reg2] -> regs[reg] ^= regs[reg2]
• 0x04 = cmp: [04][reg][imm] -> flag_zero = (regs[reg] == imm)
• 0x05 = set pc: [05][addr] -> ProgramCounter = addr
• 0x06 = jnz: [06][addr] -> if (!flag_zero) ProgramCounter = addr
• 0x07 = print: [07][reg] -> putchar(regs[reg])
• 0x08 = input: [08][reg] -> regs[reg] = getchar() (read one byte)

The given instructions inside "vmme.bin" itself is a basic XOR decrypt and then input compare, essentially a XOR decrypt + flagchecker. And after I enslaved an LLM to find me the flag inside the instructions, I got this

PO- PO- PO- PWNED!!! Not really proud of this but it is what it is.