-BIGNUM *
-auth_rsa_generate_challenge(Key *key)
-{
- BIGNUM *challenge;
- BN_CTX *ctx;
-
- if ((challenge = BN_new()) == NULL)
- fatal("auth_rsa_generate_challenge: BN_new() failed");
- /* Generate a random challenge. */
- if (BN_rand(challenge, 256, 0, 0) == 0)
- fatal("auth_rsa_generate_challenge: BN_rand failed");
- if ((ctx = BN_CTX_new()) == NULL)
- fatal("auth_rsa_generate_challenge: BN_CTX_new failed");
- if (BN_mod(challenge, challenge, key->rsa->n, ctx) == 0)
- fatal("auth_rsa_generate_challenge: BN_mod failed");
- BN_CTX_free(ctx);
-
- return challenge;
-}
-
-int
-auth_rsa_verify_response(Key *key, BIGNUM *challenge, u_char response[16])
-{
- u_char buf[32], mdbuf[16];
- MD5_CTX md;
- int len;
-
- /* don't allow short keys */
- if (BN_num_bits(key->rsa->n) < SSH_RSA_MINIMUM_MODULUS_SIZE) {
- error("auth_rsa_verify_response: RSA modulus too small: %d < minimum %d bits",
- BN_num_bits(key->rsa->n), SSH_RSA_MINIMUM_MODULUS_SIZE);
- return (0);
- }
-
- /* The response is MD5 of decrypted challenge plus session id. */
- len = BN_num_bytes(challenge);
- if (len <= 0 || len > 32)
- fatal("auth_rsa_verify_response: bad challenge length %d", len);
- memset(buf, 0, 32);
- BN_bn2bin(challenge, buf + 32 - len);
- MD5_Init(&md);
- MD5_Update(&md, buf, 32);
- MD5_Update(&md, session_id, 16);
- MD5_Final(mdbuf, &md);
-
- /* Verify that the response is the original challenge. */
- if (memcmp(response, mdbuf, 16) != 0) {
- /* Wrong answer. */
- return (0);
- }
- /* Correct answer. */
- return (1);
-}
-