- * Performs the RSA authentication dialog with the client. This returns
- * 0 if the client could not be authenticated, and 1 if authentication was
- * successful. This may exit if there is a serious protocol violation.
+ * Performs the RSA authentication challenge-response dialog with the client,
+ * and returns true (non-zero) if the client gave the correct answer to
+ * our challenge; returns zero if the client gives a wrong answer.
+ */
+
+int
+auth_rsa_challenge_dialog(Key *key)
+{
+ BIGNUM *challenge, *encrypted_challenge;
+ u_char response[16];
+ int i, success;
+
+ if ((encrypted_challenge = BN_new()) == NULL)
+ fatal("auth_rsa_challenge_dialog: BN_new() failed");
+
+ challenge = PRIVSEP(auth_rsa_generate_challenge(key));
+
+ /* Encrypt the challenge with the public key. */
+ rsa_public_encrypt(encrypted_challenge, challenge, key->rsa);
+
+ /* Send the encrypted challenge to the client. */
+ packet_start(SSH_SMSG_AUTH_RSA_CHALLENGE);
+ packet_put_bignum(encrypted_challenge);
+ packet_send();
+ BN_clear_free(encrypted_challenge);
+ packet_write_wait();
+
+ /* Wait for a response. */
+ packet_read_expect(SSH_CMSG_AUTH_RSA_RESPONSE);
+ for (i = 0; i < 16; i++)
+ response[i] = (u_char)packet_get_char();
+ packet_check_eom();
+
+ success = PRIVSEP(auth_rsa_verify_response(key, challenge, response));
+ BN_clear_free(challenge);
+ return (success);
+}
+
+/*
+ * check if there's user key matching client_n,
+ * return key if login is allowed, NULL otherwise