Adventures in armel – Debian Wheezy – udevd[XXX]: unable to receive ctrl connection: Function not implemented

Hi!

I’m running a tiny specially designed ARM system with a heavily patched 2.6.3x kernel on an armel install of Debian Squeeze. It was my goal to bring this system into the realm of security updates, and Debian Wheezy. Simple dist-upgrade right? Wrong. After the udev package was updated to the Wheezy version and I rebooted, udev immediately started spitting out this log message:

udevd[PID]: unable to receive ctrl connection: Function not implemented

Login either never started, or was just getting completely stomped by udev, because I couldn’t do so much as login to the console. I rebooted my machine in Single User mode, which succeeded. (If you’re having trouble getting into Single User mode on your Debian box, you can change your init kernel parameter to: init=/bin/bash You’ll get access to basic functions, such as file movement and such, which might save you in the event of a complete meltdown.)

I double checked the minimum kernel requirements for udev in Wheezy (2.6.18+), and double-checked that I wasn’t missing any critical udev components in the kernel. Both checked out, which didn’t leave me much to go on. (If you’re able to kill udev and login, you could strace the calls to udev during startup to see which ones are causing the failure.) After checking around online, I found a couple of threads where people had the same problem. One of them talked about lack of the accept4 syscall mapping in the kernel on ia64 devices. In that thread, there was a reference to a similar bug for armel. Hmmm.

Here‘s the ia64 thread. Here‘s the armel bug filing.

Sounds like exactly what I’m experiencing. Digging a little further, I was able to find the patch referenced there, as well as a C program that tests whether your kernel is able to use the accept4 syscall. Unfortunately, if you’ve gotten to this point, your system is quite broken, so the accept4 test might not work for you.

Here is the patch to fix 2.6.3x armel kernels to use the accept4 syscall:

From e3b4f8cdd8d2a83e8ffaed2a8f682959150365d1 Mon Sep 17 00:00:00 2001
From: Ingo Albrecht <prom@berlin.ccc.de>
Date: Sun, 18 Sep 2011 02:53:04 +0200
Subject: [PATCH 01/15] backport: wire up sys_accept4() on ARM

 * This is required to run current debian unstable (because of udev)
 * Original commit from kernel git: 21d93e2e29722d7832f61cc56d73fb953ee6578e
---
 arch/arm/include/asm/unistd.h |    1 +
 arch/arm/kernel/calls.S       |    1 +
 2 files changed, 2 insertions(+), 0 deletions(-)

diff --git a/arch/arm/include/asm/unistd.h b/arch/arm/include/asm/unistd.h
index cf9cdaa..8f32b6b 100644
--- a/arch/arm/include/asm/unistd.h
+++ b/arch/arm/include/asm/unistd.h
@@ -392,6 +392,7 @@
 #define __NR_rt_tgsigqueueinfo    	(__NR_SYSCALL_BASE+363)
 #define __NR_perf_event_open		(__NR_SYSCALL_BASE+364)
 #define __NR_recvmmsg			(__NR_SYSCALL_BASE+365)
+#define __NR_accept4			(__NR_SYSCALL_BASE+366)
 
 /*
  * The following SWIs are ARM private.
diff --git a/arch/arm/kernel/calls.S b/arch/arm/kernel/calls.S
index 9314a2d..1dff6a0 100644
--- a/arch/arm/kernel/calls.S
+++ b/arch/arm/kernel/calls.S
@@ -375,6 +375,7 @@
 		CALL(sys_rt_tgsigqueueinfo)
 		CALL(sys_perf_event_open)
 /* 365 */	CALL(sys_recvmmsg)
+		CALL(sys_accept4)
 #ifndef syscalls_counted
 .equ syscalls_padding, ((NR_syscalls + 3) & ~3) - NR_syscalls
 #define syscalls_counted
-- 
1.7.6.3

Here is the test_accept4.c program:

/* test_accept4.c

  Copyright (C) 2008, Linux Foundation, written by Michael Kerrisk
       <mtk.manpages@gmail.com>

  Licensed under the GNU GPLv2 or later.
*/
#define _GNU_SOURCE
#include <unistd.h>
#include <sys/syscall.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <stdlib.h>
#include <fcntl.h>
#include <stdio.h>
#include <string.h>

#define PORT_NUM 33333

#define die(msg) do { perror(msg); exit(EXIT_FAILURE); } while (0)

/**********************************************************************/

static int
do_test(int lfd, struct sockaddr_in *conn_addr,
       int closeonexec_flag, int nonblock_flag)
{
   int connfd, acceptfd;
   int fdf, flf, fdf_pass, flf_pass;
   struct sockaddr_in claddr;
   socklen_t addrlen;

   printf("=======================================\n");

   connfd = socket(AF_INET, SOCK_STREAM, 0);
   if (connfd == -1)
       die("socket");
   if (connect(connfd, (struct sockaddr *) conn_addr,
               sizeof(struct sockaddr_in)) == -1)
       die("connect");

   addrlen = sizeof(struct sockaddr_in);
   acceptfd = accept4(lfd, (struct sockaddr *) &claddr, &addrlen,
                      closeonexec_flag | nonblock_flag);
   if (acceptfd == -1) {
       perror("accept4()");
       close(connfd);
       return 0;
   }

   fdf = fcntl(acceptfd, F_GETFD);
   if (fdf == -1)
       die("fcntl:F_GETFD");
   fdf_pass = ((fdf & FD_CLOEXEC) != 0) ==
              ((closeonexec_flag & SOCK_CLOEXEC) != 0);
   printf("Close-on-exec flag is %sset (%s); ",
           (fdf & FD_CLOEXEC) ? "" : "not ",
           fdf_pass ? "OK" : "failed");

   flf = fcntl(acceptfd, F_GETFL);
   if (flf == -1)
       die("fcntl:F_GETFD");
   flf_pass = ((flf & O_NONBLOCK) != 0) ==
              ((nonblock_flag & SOCK_NONBLOCK) !=0);
   printf("nonblock flag is %sset (%s)\n",
           (flf & O_NONBLOCK) ? "" : "not ",
           flf_pass ? "OK" : "failed");

   close(acceptfd);
   close(connfd);

   printf("Test result: %s\n", (fdf_pass && flf_pass) ? "PASS" : "FAIL");
   return fdf_pass && flf_pass;
}

static int
create_listening_socket(int port_num)
{
   struct sockaddr_in svaddr;
   int lfd;
   int optval;

   memset(&svaddr, 0, sizeof(struct sockaddr_in));
   svaddr.sin_family = AF_INET;
   svaddr.sin_addr.s_addr = htonl(INADDR_ANY);
   svaddr.sin_port = htons(port_num);

   lfd = socket(AF_INET, SOCK_STREAM, 0);
   if (lfd == -1)
       die("socket");

   optval = 1;
   if (setsockopt(lfd, SOL_SOCKET, SO_REUSEADDR, &optval,
                  sizeof(optval)) == -1)
       die("setsockopt");

   if (bind(lfd, (struct sockaddr *) &svaddr,
            sizeof(struct sockaddr_in)) == -1)
       die("bind");

   if (listen(lfd, 5) == -1)
       die("listen");

   return lfd;
}

int
main(int argc, char *argv[])
{
   struct sockaddr_in conn_addr;
   int lfd;
   int port_num;
   int passed;

   passed = 1;

   port_num = (argc > 1) ? atoi(argv[1]) : PORT_NUM;

   memset(&conn_addr, 0, sizeof(struct sockaddr_in));
   conn_addr.sin_family = AF_INET;
   conn_addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
   conn_addr.sin_port = htons(port_num);

   lfd = create_listening_socket(port_num);

   if (!do_test(lfd, &conn_addr, 0, 0))
       passed = 0;
   if (!do_test(lfd, &conn_addr, SOCK_CLOEXEC, 0))
       passed = 0;
   if (!do_test(lfd, &conn_addr, 0, SOCK_NONBLOCK))
       passed = 0;
   if (!do_test(lfd, &conn_addr, SOCK_CLOEXEC, SOCK_NONBLOCK))
       passed = 0;

   close(lfd);

   exit(passed ? EXIT_SUCCESS : EXIT_FAILURE);
}

Just to be clear, these aren’t my patches, I simply found them, and I’m mirroring them for historical purposes. All credit goes to the original authors referenced in both snippets. And, that’s that! Apply the patch, rebuild your kernel, and reboot into your Wheezy installation. Udev should now work like a charm.

If you’d like to do this the opposite way, you could patch the udev source with the following patch found here: (But, I have not personally tested this — This will also make future updates a pain.)

diff -Nru udev-177.orig/src/udev-ctrl.c udev-177/src/udev-ctrl.c
--- udev-177.orig/src/udev-ctrl.c    2012-01-10 01:43:22.125518772 +0100
+++ udev-177/src/udev-ctrl.c	2012-01-22 16:46:31.339378651 +0100
@@ -15,6 +15,7 @@
 #include <stddef.h>
 #include <string.h>
 #include <unistd.h>
+#include <fcntl.h>
 #include <sys/types.h>
 #include <sys/poll.h>
 #include <sys/socket.h>
@@ -182,6 +183,7 @@
         struct ucred ucred;
         socklen_t slen;
         const int on = 1;
+        int flgs;
 
         conn = calloc(1, sizeof(struct udev_ctrl_connection));
         if (conn == NULL)
@@ -189,13 +191,18 @@
         conn->refcount = 1;
         conn->uctrl = uctrl;
 
-        conn->sock = accept4(uctrl->sock, NULL, NULL, SOCK_CLOEXEC|SOCK_NONBLOCK);
+        conn->sock = accept(uctrl->sock, NULL, NULL);
         if (conn->sock < 0) {
                 if (errno != EINTR)
                         err(uctrl->udev, "unable to receive ctrl connection: %m\n");
                 goto err;
         }
 
+        /* Since we don't have accept4 */
+        flgs = fcntl(conn->sock, F_GETFL, NULL);
+        if (flgs >= 0) fcntl(conn->sock, F_SETFL, flgs | O_NONBLOCK);
+        fcntl(conn->sock, F_SETFD, FD_CLOEXEC);
+
         /* check peer credential of connection */
         slen = sizeof(ucred);
         if (getsockopt(conn->sock, SOL_SOCKET, SO_PEERCRED, &ucred, &slen) < 0) {

UNIX Domain Sockets (Datagram)

Hello friends!

I am here in Internetland to talk to you about my adventures with UNIX domain sockets. I’m designing a collection of separate programs that all need to communicate. The communication has to be multiclient, single server, and easy to implement. In this case, a datagram is the preferred mode due to it’s tiny connectionless nature. UDP seems like a good option at first, but it has problems with reliability. Dropping packets in stateful IPC can cause issues with synchronization, etc. TCP could work, but why use the networking stack when there are APIs dedicated to IPC?

Sounds like a job for UNIX domain sockets. These wonderful little things use the file system for their endpoint identifiers, but the actual communication happens within the kernel. You can configure them as a pipe, or use datagrams. With datagrams, it’s in a single packet context, rather than a stream. There’s no established connection to speak of (but you do call connect()). A bit confusing. You create a file handle for your client and connect to the server’s file handle. If there’s already a file on the filesystem with the same name, the socket’s bind will fail, so cleaning the handles up when done (or before you init) is important too.

If you’re just getting into IPC, then I highly suggest reading Beej’s Guide To Unix Interprocess Communication. It’s great. I should say, there are many different types of IPC, and domain sockets are just one. I prefer domain sockets to shared memory, for the sake of simplicity. For your implementation, look at the many methods available, and see what works best for you.

Without further ado, here’s the code! This first part is the server.

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>

char * server_filename = "/tmp/socket-server";

int main(void)
{
    int s;
    struct sockaddr_un srv_un = {0};

    if ((s = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1) {
        perror("socket server");
        exit(1);
    }

    srv_un.sun_family = AF_UNIX;
    strncpy(srv_un.sun_path, server_filename, sizeof(srv_un.sun_path));
    /*If you leave the file behind when you're finished, or perhaps crash after binding, the next bind will fail
    / with "address in use". Which just means, the file is already there.*/
    unlink(srv_un.sun_path);

    if (bind(s, (struct sockaddr *)&srv_un, sizeof(srv_un)) == -1) {
        perror("bind server");
        exit(1);
    }

    for(;;) {

        char buf[1024] = {0};
        read(s, buf, sizeof(buf));
        printf("RECEIVED: %s", buf);

    }

    close(s);

    return 0;
}

And, here’s the client.

#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/un.h>

char * server_filename = "/tmp/socket-server";
char * client_filename = "/tmp/socket-client";

int main(void)
{

    int s;
    char obuf[100];
    struct sockaddr_un srv_un, cli_un = { 0 };
    
    srv_un.sun_family = AF_UNIX;
    strncpy(srv_un.sun_path, server_filename, sizeof(srv_un.sun_path));

    cli_un.sun_family = AF_UNIX;
    strncpy(cli_un.sun_path, client_filename, sizeof(cli_un.sun_path));
    unlink(cli_un.sun_path);

    if ((s = socket(AF_UNIX, SOCK_DGRAM, 0)) == -1) {
        perror("socket server");
        exit(1);
    }

    /* (http://stackoverflow.com/questions/3324619/unix-domain-socket-using-datagram-communication-between-one-server-process-and)
    Here, we bind to our client node, and connect to the server node. As Unix domain sockets need to have endpoints on either end
    of the connection. For more info, visit the URL.*/
    if (bind(s, (struct sockaddr *)&cli_un, sizeof(cli_un)) == -1) {
        perror("bind client");
        exit(1);
    }

    if (connect(s, (struct sockaddr *) &srv_un, sizeof(srv_un)) == -1) {
        perror("connect client");
        exit(1);
    }

    //printf("Connected.\n");

    while(printf("> "), fgets(obuf, 100, stdin), !feof(stdin)) {
        if (send(s, obuf, strlen(obuf), 0) == -1) {
            perror("send");
            exit(1);
        }
        break;
    }

    //printf("Sent successfully.\n");

    close(s);

    return 0;
}