Netlink : Multicast

In this program, you are going to learn

How to communication between kernel and user space ?

How to use user space APIs ?
- FD_ZERO
- FD_SET
- FD_ISSET
- NLMSG_SPACE

How to use kernel space APIs ?

Topics in this section,

User program : nl_user.c

Kernel program : nl_kernel.c

Makefile

Compile and Load

Summary

nl_user.c

To create a socket with socket(),

sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_USER);

bind() is used to bind a name to a socket

retval = bind(sock_fd, (struct sockaddr * )&src_addr, sizeof(src_addr));

nl_pid field of the sockaddr_nl can be filled with the calling process’ own pid.

struct sockaddr_nl src_addr;

src_addr.nl_pid = getpid();

Sending a Netlink Message In order to send a netlink message to the kernel or other user-space processes, another struct sockaddr_nl dest_addr needs to be supplied as the destination address, the same as sending a packet with sendmsg(). If the message is destined for the kernel, both nl_pid and nl_groups should be supplied with 0.

struct sockaddr_nl dest_addr;

dest_addr.nl_pid = 0;
dest_addr.nl_groups = 0;

struct msghdr msg;

msg.msg_name = (void * )&dest_addr;
msg.msg_namelen = sizeof(dest_addr);

The netlink socket requires its own message header as well. This is for providing a common ground for netlink messages of all protocol types. Because the Linux kernel netlink core assumes the existence of the following header in each netlink message, an application must supply this header in each netlink message it sends:

struct nlmsghdr * nlh;
nlh = (struct nlmsghdr * )malloc(NLMSG_SPACE(256));

nlh->nlmsg_len = NLMSG_SPACE(256);
nlh->nlmsg_pid = getpid();
nlh->nlmsg_flags = 0;

A netlink message thus consists of nlmsghdr and the message payload. Once a message has been entered, it enters a buffer pointed to by the nlh pointer. We also can send the message to the struct msghdr msg:

struct iovec iov;

iov.iov_base = (void * )nlh;
iov.iov_len = nlh->nlmsg_len;

msg.msg_iov = &iov;
msg.msg_iovlen = 1;

After the above steps, a call to sendmsg() kicks out the netlink message:

retval = sendmsg(sock_fd, &msg, 0);

recvmsg used for Receiving Netlink Messages,

retval = recvmsg(sock_fd, &msg, 0);

select() function indicates which of the specified file descriptors is ready for reading, ready for writing, or has an error condition pending. If the specified condition is false for all of the specified file descriptors, select() blocks, up to the specified timeout interval, until the specified condition is true for at least one of the specified file descriptors or until a signal arrives that needs to be delivered.

retval = select(sock_fd + 1, &read_fds, NULL, NULL, NULL);

FD_ZERO initializes the file descriptor set fdset to have zero bits for all file descriptors.

FD_ZERO(&read_fds);

FD_SET sets the bit for the file descriptor fd in the file descriptor set fdset.

FD_SET(sock_fd, &read_fds);

FD_ISSET returns a non-zero value if the bit for the file descriptor fd is set in the file descriptor set pointed to by fdset, and 0 otherwise.

FD_ISSET(sock_fd, &read_fds);

See the full program below,

#include <linux/netlink.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <sys/select.h>

#define NETLINK_USER 25
#define MAX_COUNT 40

int main(void)
{
	struct sockaddr_nl src_addr, dest_addr;
	struct nlmsghdr *nlh;
	struct msghdr msg;
	struct iovec iov;
	char recv_buff[256];
	int sock_fd, retval, count;

	sock_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_USER);
	if (sock_fd < 0) {
		perror("socket");
		return -1;
	}

	memset(&src_addr, 0, sizeof(src_addr));
	src_addr.nl_family = AF_NETLINK;
	src_addr.nl_pid = getpid(); // User-space process PID
	src_addr.nl_groups = 0; // Unicast

	retval = bind(sock_fd, (struct sockaddr *)&src_addr, sizeof(src_addr));
	if (retval < 0) {
		perror("bind");
		close(sock_fd);
		return -1;
	}

	memset(&dest_addr, 0, sizeof(dest_addr));
	memset(&msg, 0, sizeof(msg));

	dest_addr.nl_family = AF_NETLINK;
	dest_addr.nl_pid = 0; // To kernel
	dest_addr.nl_groups = 0;


	fd_set read_fds;

	FD_ZERO(&read_fds);
	FD_SET(sock_fd, &read_fds);

	count = 0;
	while (count < MAX_COUNT) {
		nlh = (struct nlmsghdr *)malloc(NLMSG_SPACE(256));
		if (!nlh) {
			perror("malloc");
			close(sock_fd);
			return -1;
		}

		nlh->nlmsg_len = NLMSG_SPACE(256);
		nlh->nlmsg_pid = getpid();
		nlh->nlmsg_flags = 0;

		iov.iov_base = (void *)nlh;
		iov.iov_len = nlh->nlmsg_len;
		msg.msg_name = (void *)&dest_addr;
		msg.msg_namelen = sizeof(dest_addr);
		msg.msg_iov = &iov;
		msg.msg_iovlen = 1;


		snprintf(NLMSG_DATA(nlh), 256, "Message from user (send) %d", count);

		retval = sendmsg(sock_fd, &msg, 0);
		if (retval == -1) {
			perror("sendmsg");
			break;
		}

		retval = select(sock_fd + 1, &read_fds, NULL, NULL, NULL);
		if (retval == -1) {
			perror("select");
			break;
		} else if (retval) {
			if (FD_ISSET(sock_fd, &read_fds)) {
				memset(recv_buff, 0, sizeof(recv_buff));
				retval = recvmsg(sock_fd, &msg, 0);
				if (retval == -1) {
					perror("recv");
					break;
				}
				printf("Received from kernel: %s\n", (char *) NLMSG_DATA(nlh));
			}
		}
		count++;
	}

	free(nlh);
	close(sock_fd);

	return 0;
}

nl_kernel.c

init_waitqueue_head are declared and initialized.

init_waitqueue_head(&wq_r);
init_waitqueue_head(&wq_s);

wait_event_interruptible is put to sleep (TASK_INTERRUPTIBLE) until the condition evaluates to true.

wait_event_interruptible(wq_s, wq_flag != 0);
wait_event_interruptible(wq_r, wq_flag != 0);

wake_up_interruptible wakes up only the processes that are in interruptible sleeps. Any process that sleeps on the wait queue using a noninterruptible function or macro will continue to sleep.

wake_up_interruptible(&wq_s);
wake_up_interruptible(&wq_r);

kthread_run is used to create and start a kernel thread.

thread = kthread_run(recv_netlink_msg_thread, NULL, "netlink_kthread");
send_thread = kthread_run(send_netlink_msg_thread, NULL, "netlink_send_kthread");

kthread_stop is used to stop and clean up a kernel thread created with kthread_run.

kthread_stop(thread);
kthread_stop(send_thread);

kthread_should_stop is to determine when thread should exit.

while (!kthread_should_stop())

INIT_LIST_HEAD is used to initialize a list_head structure.

INIT_LIST_HEAD(&mesg->list);
INIT_LIST_HEAD(&msg->list);
INIT_LIST_HEAD(&send_list);

list_head is used to initialize the list.

struct list_head(msg_list);
struct list_head(send_list);

list_for_each_entry is used to iterate over list of given type.

list_for_each_entry(new_node, &send_list, list)
list_for_each_entry(mesg, &send_list, list)
list_for_each_entry(new_node, &msg_list, list)
list_for_each_entry(msg, &msg_list, list)

list_for_each_entry_safe is used to iterate over list of given type safe against removal of list entry.

list_for_each_entry_safe(curr_node, next_node, &msg_list, list)
list_for_each_entry_safe(curr_node, next_node, &send_list, list)

list_add_tail is used to add a new entry.

list_add_tail(&mesg->list, &send_list);
list_add_tail(&msg->list, &msg_list);

list_del is used to delete entry from list.

list_del(&mesg->list);
list_del(&msg->list);
list_del(&curr_node->list);
list_del(&curr_node->list);

nlmsg_new using this create a netlink message.

skb_out = nlmsg_new(msg_size, 0);

nlmsg_put used to populate the message with data.

nlh = nlmsg_put(skb_out, 0, 0, NLMSG_DONE, msg_size, 0);

nlmsg_unicast used to send the message to the user space application.

res = nlmsg_unicast(socket, skb_out, pid);

netlink_kernel_release used to release the netlink socket created with netlink_kernel_create.

netlink_kernel_release(socket);

See the full program below,

#include <linux/module.h>
#include <linux/init.h>
#include <linux/skbuff.h>
#include <linux/kthread.h>
#include <linux/sched.h>
#include <linux/delay.h>
#include <linux/netlink.h>
#include <net/netlink.h>
#include <net/net_namespace.h>
#include <linux/list.h>
#include <linux/slab.h>

#define NETLINK_TESTFAMILY 25
#define MAX_PAYLOAD 1024

struct message {
	char data[MAX_PAYLOAD];
	struct list_head list;
};

struct list_head(msg_list);
struct list_head(send_list);

wait_queue_head_t wq_r;
wait_queue_head_t wq_s;

int wq_flag;

static struct task_struct *thread;
static struct task_struct *send_thread;
struct sock *socket;

static int send_netlink_msg_thread(void *arg)
{
	struct message *new_node;

	while (!kthread_should_stop()) {
		wait_event_interruptible(wq_s, wq_flag != 0);
		if (wq_flag == 1) {
			list_for_each_entry(new_node, &send_list, list) {
				pr_info("kernel thread-1 : Sent message :  %s\n", new_node->data);
			}
		}

		if (wq_flag == 2) {
			pr_info("Event came from exit function\n");
			return 0;
		}
		wq_flag = 0;
		msleep(5000);
	}
	return 0;
}

int send_netlink_msg(int pid, char *msg, int msg_size)
{
	struct sk_buff *skb_out;
	struct nlmsghdr *nlh;
	int res;
	struct message *mesg;
	int count = 0;

	skb_out = nlmsg_new(msg_size, 0);
	if (!skb_out) {
		pr_err("Failed to allocate new skb\n");
		return -1;
	}

	nlh = nlmsg_put(skb_out, 0, 0, NLMSG_DONE, msg_size, 0);
	if (nlh == NULL) {
		pr_err("nlmsg_put failaure\n");
		nlmsg_free(skb_out);
		return -1;
	}

	NETLINK_CB(skb_out).dst_group = 0;
	strncpy(nlmsg_data(nlh), msg, msg_size);

	res = nlmsg_unicast(socket, skb_out, pid);
	if (res < 0) {
		pr_err("Error while sending the netlink msg to the user\n");
		return res;
	}

	mesg = kmalloc(sizeof(struct message), GFP_KERNEL);

	strncpy(mesg->data, nlmsg_data(nlh), MAX_PAYLOAD);

	INIT_LIST_HEAD(&mesg->list);
	list_add_tail(&mesg->list, &send_list);

	list_for_each_entry(mesg, &send_list, list) {
		count++;
		if (count > 50) {
			list_del(&mesg->list);
			kfree(mesg);
		}
	}

	wq_flag = 2;
	wake_up_interruptible(&wq_s);

	return 0;
}


static int recv_netlink_msg_thread(void *arg)
{
	struct message *new_node;

	while (!kthread_should_stop()) {
		wait_event_interruptible(wq_r, wq_flag != 0);
		if (wq_flag == 1) {
			list_for_each_entry(new_node, &msg_list, list) {
				pr_info("kernel thread-1 : Received message :  %s\n", new_node->data);
			}
		}

		if (wq_flag == 2) {
			pr_info("Event came from exit function\n");
			return 0;
		}
		wq_flag = 0;
		msleep(5000);
	}
	return 0;
}


static void recv_netlink_msg(struct sk_buff *skb)
{
	struct nlmsghdr *nlh;
	struct message *msg;
	int pid;
	int count = 0;

	nlh = (struct nlmsghdr *) skb->data;
	msg = kmalloc(sizeof(struct message), GFP_KERNEL);
	if (!msg) {
		pr_err("Failed to allocate memory\n");
		return;
	}
	pid = nlh->nlmsg_pid;
	strscpy(msg->data, nlmsg_data(nlh), MAX_PAYLOAD);

	INIT_LIST_HEAD(&msg->list);
	list_add_tail(&msg->list, &msg_list);

	list_for_each_entry(msg, &msg_list, list) {
		count++;
		if (count > 50) {
			list_del(&msg->list);
			kfree(msg);
		}
	}

	send_netlink_msg(pid, "Hello from kernel", strlen("Hello from kernel"));
	wq_flag = 1;
	wake_up_interruptible(&wq_r);

}


static int __init netlink_init(void)
{
	struct netlink_kernel_cfg cfg = {
		.input = recv_netlink_msg,
	};

	INIT_LIST_HEAD(&msg_list);
	INIT_LIST_HEAD(&send_list);

	socket = netlink_kernel_create(&init_net, NETLINK_TESTFAMILY, &cfg);
	if (socket == NULL)
		return -1;

	init_waitqueue_head(&wq_r);
	init_waitqueue_head(&wq_s);

	thread = kthread_run(recv_netlink_msg_thread, NULL, "netlink_kthread");
	if (IS_ERR(thread)) {
		pr_alert("Error in creating kernel thread\n");
		netlink_kernel_release(socket);
		return PTR_ERR(thread);
	}

	send_thread = kthread_run(send_netlink_msg_thread, NULL, "netlink_send_kthread");
	if (IS_ERR(send_thread)) {
		pr_alert("Error in creating kernel thread\n");
		netlink_kernel_release(socket);
		return PTR_ERR(send_thread);
	}

	pr_info("Netlink kernel module initialized\n");
	return 0;
}

static void __exit netlink_exit(void)
{
	struct message *curr_node, *next_node;

	wq_flag = 2;
	wake_up_interruptible(&wq_r);
	wake_up_interruptible(&wq_s);

	if (thread) {
		kthread_stop(thread);
		thread = NULL;
	}

	if (send_thread) {
		kthread_stop(send_thread);
		send_thread = NULL;
	}

	list_for_each_entry_safe(curr_node, next_node, &msg_list, list) {
		list_del(&curr_node->list);
		kfree(curr_node);
	}

	list_for_each_entry_safe(curr_node, next_node, &send_list, list) {
		list_del(&curr_node->list);
		kfree(curr_node);
	}

	if (socket)
		netlink_kernel_release(socket);

	pr_info("Netlink kernel module exited\n");
}

module_init(netlink_init);
module_exit(netlink_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Linux_usr");
MODULE_DESCRIPTION("Netlink");

Makefile

obj-m += nl_kernel.o
all:
	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) modules

client:
	gcc nl_user.c -o nl_user

clean:
	make -C /lib/modules/$(shell uname -r)/build M=$(PWD) clean
	rm nl_user

Compile and Load

$ make all

$ sudo insmod ./nl_kernel.ko

$ ./nl_user
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel
Received from kernel: Hello from kernel

$ sudo rmmod nl_kernel

$ dmesg
[ 2066.487382] Netlink kernel module initialized
[ 2079.443801] kernel thread-1 : Sent message :  Hello from kernel
[ 2079.443900] kernel thread-1 : Received message :  Message from user (send) 0
[ 2079.443904] kernel thread-1 : Received message :  Message from user (send) 1
[ 2084.449653] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449656] kernel thread-1 : Received message :  Message from user (send) 0
[ 2084.449659] kernel thread-1 : Received message :  Message from user (send) 1
[ 2084.449660] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449662] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449663] kernel thread-1 : Received message :  Message from user (send) 2
[ 2084.449665] kernel thread-1 : Received message :  Message from user (send) 3
[ 2084.449667] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449668] kernel thread-1 : Received message :  Message from user (send) 4
[ 2084.449670] kernel thread-1 : Received message :  Message from user (send) 5
[ 2084.449672] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449673] kernel thread-1 : Received message :  Message from user (send) 6
[ 2084.449675] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449677] kernel thread-1 : Received message :  Message from user (send) 7
[ 2084.449679] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449681] kernel thread-1 : Received message :  Message from user (send) 8
[ 2084.449683] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449685] kernel thread-1 : Received message :  Message from user (send) 9
[ 2084.449686] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449688] kernel thread-1 : Received message :  Message from user (send) 10
[ 2084.449690] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449692] kernel thread-1 : Received message :  Message from user (send) 11
[ 2084.449694] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449696] kernel thread-1 : Received message :  Message from user (send) 12
[ 2084.449698] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449699] kernel thread-1 : Received message :  Message from user (send) 13
[ 2084.449701] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449703] kernel thread-1 : Received message :  Message from user (send) 14
[ 2084.449704] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449706] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449708] kernel thread-1 : Received message :  Message from user (send) 15
[ 2084.449710] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449712] kernel thread-1 : Received message :  Message from user (send) 16
[ 2084.449714] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449716] kernel thread-1 : Received message :  Message from user (send) 17
[ 2084.449717] kernel thread-1 : Received message :  Message from user (send) 18
[ 2084.449719] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449721] kernel thread-1 : Received message :  Message from user (send) 19
[ 2084.449723] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449725] kernel thread-1 : Received message :  Message from user (send) 20
[ 2084.449726] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449728] kernel thread-1 : Received message :  Message from user (send) 21
[ 2084.449730] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449732] kernel thread-1 : Received message :  Message from user (send) 22
[ 2084.449733] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449735] kernel thread-1 : Received message :  Message from user (send) 23
[ 2084.449737] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449739] kernel thread-1 : Received message :  Message from user (send) 24
[ 2084.449741] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449743] kernel thread-1 : Received message :  Message from user (send) 25
[ 2084.449745] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449746] kernel thread-1 : Received message :  Message from user (send) 26
[ 2084.449748] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449750] kernel thread-1 : Received message :  Message from user (send) 27
[ 2084.449752] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449754] kernel thread-1 : Received message :  Message from user (send) 28
[ 2084.449756] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449758] kernel thread-1 : Received message :  Message from user (send) 29
[ 2084.449760] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449762] kernel thread-1 : Received message :  Message from user (send) 30
[ 2084.449764] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449766] kernel thread-1 : Received message :  Message from user (send) 31
[ 2084.449767] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449769] kernel thread-1 : Received message :  Message from user (send) 32
[ 2084.449771] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449773] kernel thread-1 : Received message :  Message from user (send) 33
[ 2084.449774] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449776] kernel thread-1 : Received message :  Message from user (send) 34
[ 2084.449778] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449780] kernel thread-1 : Received message :  Message from user (send) 35
[ 2084.449781] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449784] kernel thread-1 : Received message :  Message from user (send) 36
[ 2084.449786] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449787] kernel thread-1 : Received message :  Message from user (send) 37
[ 2084.449789] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449791] kernel thread-1 : Received message :  Message from user (send) 38
[ 2084.449793] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449795] kernel thread-1 : Received message :  Message from user (send) 39
[ 2084.449796] kernel thread-1 : Sent message :  Hello from kernel
[ 2084.449798] kernel thread-1 : Sent message :  Hello from kernel

Summary

User Space API	Learning
socket	To create a socket
bind	To bind a name to a socket
select	To monitor the activity on a set of sockets looking for sockets ready for reading, writing, or with an exception condition pending
FD_ZERO	To initialize the file descriptor set fdset to have zero bits for all file descriptors
FD_SET	To set the bit for the file descriptor fd in the file descriptor set fdset
FD_ISSET	It returns a non-zero value if the bit for the file descriptor fd is set in the file descriptor set pointed to by fdset, and 0 otherwise
sendmsg	To send netlink message
recvmsg	To receive netlink message

Summary

Kernel Space API	Learning
init_waitqueue_head	To declare and initialize
wait_event_interruptible	It sleep until a condition gets true
wake_up_interruptible	It wakes up only the processes that are in interruptible sleeps
kthread_run	To create and start a kernel thread
kthread_stop	To stop and clean up a kernel thread created with kthread_run
kthread_should_stop	To determine when thread should exit
INIT_LIST_HEAD	To initialize a list_head structure
list_head	To initialize the list
list_for_each_entry	To iterate over list of given type
list_for_each_entry_safe	To iterate over list of given type safe against removal of list entry
list_add_tail	To add a new entry
list_del	To delete entry from list
nlmsg_new	To create a netlink message
nlmsg_put	TO populate the message
nlmsg_unicast	To send the message to the user space application
netlink_kernel_release	To release the netlink socket