+BIGNUM *
+auth_rsa_generate_challenge(Key *key)
+{
+ BIGNUM *challenge;
+ BN_CTX *ctx;
+
+ if ((challenge = BN_new()) == NULL)
+ fatal("auth_rsa_generate_challenge: BN_new() failed");
+ /* Generate a random challenge. */
+ if (BN_rand(challenge, 256, 0, 0) == 0)
+ fatal("auth_rsa_generate_challenge: BN_rand failed");
+ if ((ctx = BN_CTX_new()) == NULL)
+ fatal("auth_rsa_generate_challenge: BN_CTX_new failed");
+ if (BN_mod(challenge, challenge, key->rsa->n, ctx) == 0)
+ fatal("auth_rsa_generate_challenge: BN_mod failed");
+ BN_CTX_free(ctx);
+
+ return challenge;
+}
+
+int
+auth_rsa_verify_response(Key *key, BIGNUM *challenge, u_char response[16])
+{
+ u_char buf[32], mdbuf[16];
+ MD5_CTX md;
+ int len;
+
+ /* don't allow short keys */
+ if (BN_num_bits(key->rsa->n) < SSH_RSA_MINIMUM_MODULUS_SIZE) {
+ error("auth_rsa_verify_response: RSA modulus too small: %d < minimum %d bits",
+ BN_num_bits(key->rsa->n), SSH_RSA_MINIMUM_MODULUS_SIZE);
+ return (0);
+ }
+
+ /* The response is MD5 of decrypted challenge plus session id. */
+ len = BN_num_bytes(challenge);
+ if (len <= 0 || len > 32)
+ fatal("auth_rsa_verify_response: bad challenge length %d", len);
+ memset(buf, 0, 32);
+ BN_bn2bin(challenge, buf + 32 - len);
+ MD5_Init(&md);
+ MD5_Update(&md, buf, 32);
+ MD5_Update(&md, session_id, 16);
+ MD5_Final(mdbuf, &md);
+
+ /* Verify that the response is the original challenge. */
+ if (memcmp(response, mdbuf, 16) != 0) {
+ /* Wrong answer. */
+ return (0);
+ }
+ /* Correct answer. */
+ return (1);
+}
+