handle failure in encrypt_rfbdes() in callers

include/rfb/rfbproto.h

@@ -1557,7 +1557,7 @@ typedef union {
 extern int rfbEncryptAndStorePasswd(char *passwd, char *fname);
 extern char *rfbDecryptPasswdFromFile(char *fname);
 extern void rfbRandomBytes(unsigned char *bytes);
-extern void rfbEncryptBytes(unsigned char *bytes, char *passwd);
+extern int rfbEncryptBytes(unsigned char *bytes, char *passwd);
 
 
 #endif

src/common/vncauth.c

@@ -102,7 +102,10 @@ rfbEncryptAndStorePasswd(char *passwd, char *fname)
 
     /* Do encryption in-place - this way we overwrite our copy of the plaintext
        password */
-    encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd));
+    if (encrypt_rfbdes(encryptedPasswd, &out_len, fixedkey, encryptedPasswd, sizeof(encryptedPasswd)) == 0) {
+	fclose(fp);
+	return 1;
+    }
 
     for (i = 0; i < 8; i++) {
 	putc(encryptedPasswd[i], fp);
@@ -180,7 +183,7 @@ rfbRandomBytes(unsigned char *bytes)
  * Encrypt CHALLENGESIZE bytes in memory using a password.
  */
 
-void
+int
 rfbEncryptBytes(unsigned char *bytes, char *passwd)
 {
     unsigned char key[8];
@@ -197,19 +200,29 @@ rfbEncryptBytes(unsigned char *bytes, char *passwd)
 	}
     }
 
-    encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE);
+    if (encrypt_rfbdes(bytes, &out_len, key, bytes, CHALLENGESIZE) == 0) {
+	return 1;
+    }
+    return 0;
 }
 
-void
+int
 rfbEncryptBytes2(unsigned char *where, const int length, unsigned char *key) {
   int i, j, out_len;
   for (i = 0; i< 8; i++)
     where[i] ^= key[i];
-  encrypt_rfbdes(where, &out_len, key, where, 8);
+  if (encrypt_rfbdes(where, &out_len, key, where, 8) == 0) {
+    fclose(fp);
+    return 1;
+  }
   for (i = 8; i < length; i += 8) {
     for (j = 0; j < 8; j++) {
       where[i + j] ^= where[i + j - 8];
     }
-    encrypt_rfbdes(where + i, &out_len, key, where + i, 8);
+    if (encrypt_rfbdes(where + i, &out_len, key, where + i, 8) == 0) {
+      fclose(fp);
+      return 1;
+    }
   }
+  return 0;
 }

src/libvncclient/rfbclient.c

@@ -404,8 +404,8 @@ rfbBool ConnectToRFBRepeater(rfbClient* client,const char *repeaterHost, int rep
   return TRUE;
 }
 
-extern void rfbClientEncryptBytes(unsigned char* bytes, char* passwd);
-extern void rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key);
+extern int rfbClientEncryptBytes(unsigned char* bytes, char* passwd);
+extern int rfbClientEncryptBytes2(unsigned char *where, const int length, unsigned char *key);
 
 static void
 ReadReason(rfbClient* client)
@@ -585,7 +585,10 @@ HandleVncAuth(rfbClient *client)
         passwd[8] = '\0';
       }
 
-      rfbClientEncryptBytes(challenge, passwd);
+      if (rfbClientEncryptBytes(challenge, passwd) != 0) {
+        rfbClientLog("Encryption failed\n");
+        return FALSE;
+      }
 
       /* Lose the password from memory */
       for (i = strlen(passwd); i >= 0; i--) {
@@ -733,8 +736,14 @@ HandleUltraMSLogonIIAuth(rfbClient *client)
   strncpy((char *)password, cred->userCredential.password, sizeof(password)-1);
   FreeUserCredential(cred);
 
-  rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key);
-  rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key);
+  rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key) != 0) {
+    rfbClientLog("Encrypting username failed\n");
+    return FALSE;
+  }
+  rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key) != 0) {
+    rfbClientLog("Encrypting password failed\n");
+    return FALSE;
+  }
 
   if (!WriteToRFBServer(client, (char *)pub, sizeof(pub))) return FALSE;
   if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE;
@@ -789,8 +798,14 @@ HandleMSLogonAuth(rfbClient *client)
   pub = rfbClientSwap64IfLE(pub);
   key = rfbClientSwap64IfLE(key);
 
-  rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)&key);
-  rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)&key);
+  rfbClientEncryptBytes2(username, sizeof(username), (unsigned char *)key) != 0) {
+    rfbClientLog("Encrypting username failed\n");
+    return FALSE;
+  }
+  rfbClientEncryptBytes2(password, sizeof(password), (unsigned char *)key) != 0) {
+    rfbClientLog("Encrypting password failed\n");
+    return FALSE;
+  }
 
   if (!WriteToRFBServer(client, (char *)&pub, 8)) return FALSE;
   if (!WriteToRFBServer(client, (char *)username, sizeof(username))) return FALSE;

src/libvncserver/main.c

@@ -792,7 +792,11 @@ static rfbBool rfbDefaultPasswordCheck(rfbClientPtr cl,const char* response,int
     return(FALSE);
   }
 
-  rfbEncryptBytes(cl->authChallenge, passwd);
+  if (rfbEncryptBytes(cl->authChallenge, passwd) != 0) {
+    rfbErr("Encryption failed\n");
+    free(passwd);
+    return(FALSE);
+  }
 
   /* Lose the password from memory */
   for (i = strlen(passwd); i >= 0; i--) {
@@ -820,7 +824,10 @@ rfbBool rfbCheckPasswordByList(rfbClientPtr cl,const char* response,int len)
   for(passwds=(char**)cl->screen->authPasswdData;*passwds;passwds++,i++) {
     uint8_t auth_tmp[CHALLENGESIZE];
     memcpy((char *)auth_tmp, (char *)cl->authChallenge, CHALLENGESIZE);
-    rfbEncryptBytes(auth_tmp, *passwds);
+    if (rfbEncryptBytes(auth_tmp, *passwds) != 0) {
+      rfbErr("Encryption failed\n");
+      return(FALSE);
+    }
 
     if (memcmp(auth_tmp, response, len) == 0) {
       if(i>=cl->screen->authPasswdFirstViewOnly)