ntdll: Fix the return value of NtOpenKey for some invalid parameters.

dlls/advapi32/registry.c

@@ -311,6 +311,7 @@ LSTATUS WINAPI RegOpenKeyExW( HKEY hkey, LPCWSTR name, DWORD options, REGSAM acc
     /* NT+ allows beginning backslash for HKEY_CLASSES_ROOT */
     if (hkey == HKEY_CLASSES_ROOT && name && *name == '\\') name++;
 
+    if (!retkey) return ERROR_INVALID_PARAMETER;
     if (!(hkey = get_special_root_hkey( hkey ))) return ERROR_INVALID_HANDLE;
 
     attr.Length = sizeof(attr);

dlls/ntdll/reg.c

@@ -116,13 +116,13 @@ NTSTATUS WINAPI NtOpenKey( PHANDLE retkey, ACCESS_MASK access, const OBJECT_ATTR
     NTSTATUS ret;
     DWORD len;
 
-    if (!attr) return STATUS_ACCESS_VIOLATION;
+    if (!retkey || !attr) return STATUS_ACCESS_VIOLATION;
+    if (attr->Length > sizeof(OBJECT_ATTRIBUTES)) return STATUS_INVALID_PARAMETER;
     len = attr->ObjectName->Length;
     TRACE( "(%p,%s,%x,%p)\n", attr->RootDirectory,
            debugstr_us(attr->ObjectName), access, retkey );
 
     if (len > MAX_NAME_LENGTH) return STATUS_BUFFER_OVERFLOW;
-    if (!retkey) return STATUS_INVALID_PARAMETER;
 
     SERVER_START_REQ( open_key )
     {

dlls/ntdll/tests/reg.c

@@ -351,13 +351,11 @@ static void test_NtOpenKey(void)
 
     /* NULL key */
     status = pNtOpenKey(NULL, am, &attr);
-    todo_wine
     ok(status == STATUS_ACCESS_VIOLATION, "Expected STATUS_ACCESS_VIOLATION, got: 0x%08x\n", status);
 
     /* Length > sizeof(OBJECT_ATTRIBUTES) */
     attr.Length *= 2;
     status = pNtOpenKey(&key, am, &attr);
-    todo_wine
     ok(status == STATUS_INVALID_PARAMETER, "Expected STATUS_INVALID_PARAMETER, got: 0x%08x\n", status);
 }