- (dtucker) Bug #536: Test for and work around openpty/controlling tty

   problem on Linux (fixes "could not set controlling tty" errors).

Also renames STREAMS_PUSH_ACQUIRES_CTTY to the more generic SSHD_ACQUIRES_CTTY
and moves the Solaris-specific comments to configure.ac.

diff --git a/ChangeLog b/ChangeLog
index 6dc646be3c16653a845faede30b100ee359879dc..5b02035e2c09d5b8d09a71263a7b5f1328faa420 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,6 +1,8 @@
 20030510
  - (dtucker) Bug #318: Create ssh_prng_cmds.out during "make" rather than
    "make install".  Patch by roth@feep.net.
+ - (dtucker) Bug #536: Test for and work around openpty/controlling tty
+   problem on Linux (fixes "could not set controlling tty" errors).
 
 20030504
  - (dtucker) Bug #497: Move #include of bsd-cygwin_util.h to openbsd-compat.h.

diff --git a/acconfig.h b/acconfig.h
index b4a7955d30e40c43c9e62f9cb348f8816df4ec5d..874fcb11f4f9010be2c0a9bd8536e668edd74d52 100644 (file)
--- a/acconfig.h
+++ b/acconfig.h
@@ -374,8 +374,8 @@
 /* Some systems put this outside of libc */
 #undef HAVE_NANOSLEEP
 
-/* Pushing STREAMS modules incorrectly acquires a controlling TTY */
-#undef STREAMS_PUSH_ACQUIRES_CTTY
+/* Define if sshd somehow reacquires a controlling TTY after setsid() */
+#undef SSHD_ACQUIRES_CTTY
 
 /* Define if cmsg_type is not passed correctly */
 #undef BROKEN_CMSG_TYPE

diff --git a/configure.ac b/configure.ac
index c269b7fde8a5760c59ae47d0f260aed1be2ee024..35733ffb06af64859110caad4796f5e1ce9a80d7 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -188,15 +188,56 @@ main() { if (NSVersionOfRunTimeLibrary("System") >= (60 << 16))
        AC_DEFINE(SETPROCTITLE_STRATEGY,PS_USE_CLOBBER_ARGV)
        AC_DEFINE(SETPROCTITLE_PS_PADDING, '\0')
        inet6_default_4in6=yes
-       AC_MSG_CHECKING(for broken cmsg_type)
+       # openpty can be in libutil, needed for controlling tty test
+       AC_SEARCH_LIBS(openpty, util)
+       # make sure that openpty does not reacquire controlling terminal
+       AC_MSG_CHECKING(if openpty correctly handles controlling tty)
+       AC_TRY_RUN(
+               [
+#include <stdio.h>
+#include <sys/fcntl.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+int
+main()
+{
+       pid_t pid;
+       int fd, ptyfd, ttyfd, status;
+
+       pid = fork();
+       if (pid < 0) {          /* failed */
+               exit(1);
+       } else if (pid > 0) {   /* parent */
+               waitpid(pid, &status, 0);
+               if (WIFEXITED(status)) 
+                       exit(WEXITSTATUS(status));
+               else
+                       exit(2);
+       } else {                /* child */
+               close(0); close(1); close(2);
+               setsid();
+               openpty(&ptyfd, &ttyfd, NULL, NULL, NULL);
+               fd = open("/dev/tty", O_RDWR | O_NOCTTY);
+               if (fd >= 0)
+                       exit(3);        /* Acquired ctty: broken */
+               else
+                       exit(0);        /* Did not acquire ctty: OK */
+       }
+}
+               ],
+               [
+                       AC_MSG_RESULT(yes)
+               ],
+               [
+                       AC_MSG_RESULT(no)
+                       AC_DEFINE(SSHD_ACQUIRES_CTTY)
+               ]
+       )
        case `uname -r` in
-       2.0.*)
-               AC_MSG_RESULT(yes)
+       1.*|2.0.*)
                AC_DEFINE(BROKEN_CMSG_TYPE)
                ;;
-       *)
-               AC_MSG_RESULT(no)
-               ;;
        esac
        ;;
 mips-sony-bsd|mips-sony-newsos4)
@@ -230,7 +271,8 @@ mips-sony-bsd|mips-sony-newsos4)
        AC_DEFINE(LOGIN_NEEDS_UTMPX)
        AC_DEFINE(LOGIN_NEEDS_TERM)
        AC_DEFINE(PAM_TTY_KLUDGE)
-       AC_DEFINE(STREAMS_PUSH_ACQUIRES_CTTY)
+       # Pushing STREAMS modules will cause sshd to acquire a controlling tty.
+       AC_DEFINE(SSHD_ACQUIRES_CTTY)
        # hardwire lastlog location (can't detect it on some versions)
        conf_lastlog_location="/var/adm/lastlog"
        AC_MSG_CHECKING(for obsolete utmp and wtmp in solaris2.x)

diff --git a/sshd.c b/sshd.c
index 62b4f45f4f1cd9898152e63e49fa44645d236cdf..63070ac23f8c9d2493e5256b6212fbb22ed1575c 100644 (file)
--- a/sshd.c
+++ b/sshd.c
@@ -1400,11 +1400,11 @@ main(int ac, char **av)
         * setlogin() affects the entire process group.  We don't
         * want the child to be able to affect the parent.
         */
-#if !defined(STREAMS_PUSH_ACQUIRES_CTTY)
+#if !defined(SSHD_ACQUIRES_CTTY)
        /*
-        * If setsid is called on Solaris, sshd will acquire the controlling
-        * terminal while pushing STREAMS modules. This will prevent the
-        * shell from acquiring it later.
+        * If setsid is called, on some platforms sshd will later acquire a
+        * controlling terminal which will result in "could not set
+        * controlling tty" errors.
         */
        if (!debug_flag && !inetd_flag && setsid() < 0)
                error("setsid: %.100s", strerror(errno));