- (dtucker) [configure.ac] Bug #536: Test for and work around openpty/

   controlling tty problem on Linux (fixes "could not set controlling tty"
   errors).

diff --git a/ChangeLog b/ChangeLog
index aabe5f8a3918d35aac8f1ba900166c450ee4dade..d1b4744e9bc2cab6a76a4bd750023463f42b92ba 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -16,6 +16,9 @@
    invalid cmsg_type on Linux 1.x and 2.0 kernels, privsep should now work.
  - (dtucker) [acconfig.h configure.ac sshd.c] STREAMS_PUSH_ACQUIRES_CTTY ->
    SSHD_ACQUIRES_CTTY.
+ - (dtucker) [configure.ac] Bug #536: Test for and work around openpty/
+   controlling tty problem on Linux (fixes "could not set controlling tty"
+   errors).
 
 20030429
  - (djm) Add back radix.o (used by AFS support), after it went missing from

diff --git a/configure.ac b/configure.ac
index 60b3b656ecf07bb341733a60407d19c2fb4ecfa4..d94f588d5c2b4669422d3584de5f32f20d80a8c0 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -183,6 +183,7 @@ main() { if (NSVersionOfRunTimeLibrary("System") >= (60 << 16))
 *-*-linux*)
        no_dev_ptmx=1
        check_for_libcrypt_later=1
+       check_for_openpty_ctty_bug=1
        AC_DEFINE(DONT_TRY_OTHER_AF)
        AC_DEFINE(PAM_TTY_KLUDGE)
        AC_DEFINE(SETPROCTITLE_STRATEGY,PS_USE_CLOBBER_ARGV)
@@ -733,6 +734,52 @@ unlink(template); exit(0);
 )
 fi
 
+if test ! -z "$check_for_openpty_ctty_bug"; then
+       AC_MSG_CHECKING(if openpty correctly handles controlling tty)
+       AC_TRY_RUN(
+               [
+#include <stdio.h>
+#include <sys/fcntl.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+int
+main()
+{
+       pid_t pid;
+       int fd, ptyfd, ttyfd, status;
+
+       pid = fork();
+       if (pid < 0) {          /* failed */
+               exit(1);
+       } else if (pid > 0) {   /* parent */
+               waitpid(pid, &status, 0);
+               if (WIFEXITED(status))
+                       exit(WEXITSTATUS(status));
+               else
+                       exit(2);
+       } else {                /* child */
+               close(0); close(1); close(2);
+               setsid();
+               openpty(&ptyfd, &ttyfd, NULL, NULL, NULL);
+               fd = open("/dev/tty", O_RDWR | O_NOCTTY);
+               if (fd >= 0)
+                       exit(3);        /* Acquired ctty: broken */
+               else
+                       exit(0);        /* Did not acquire ctty: OK */
+       }
+}
+               ],
+               [
+                       AC_MSG_RESULT(yes)
+               ],
+               [
+                       AC_MSG_RESULT(no)
+                       AC_DEFINE(SSHD_ACQUIRES_CTTY)
+               ]
+       )
+fi
+
 AC_FUNC_GETPGRP
 
 # Check for PAM libs