libkfont: Check console mode

No font getting or setting operations work unless the console is in
text mode[1]. This results in a hard-to-diagnose “Invalid argument”
error.

[1] https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/drivers/tty/vt/vt.c#n4736

Link: #120
Signed-off-by: Alexey Gladkov <legion@kernel.org>

src/libkfont/kdfontop.c

@@ -21,11 +21,31 @@
 #include "compat/linux-kd.h"
 #endif
 
+static int
+is_kd_text(struct kfont_context *ctx, int fd)
+{
+	unsigned int kd_mode;
+
+	if (ioctl(fd, KDGETMODE, &kd_mode)) {
+		KFONT_ERR(ctx, "ioctl(KDGETMODE): %m");
+		return 0;
+	}
+
+	if (kd_mode == KD_TEXT)
+		return 1;
+
+	KFONT_ERR(ctx, _("Console is not in text mode"));
+	return 0;
+}
+
 int
 kfont_restore_font(struct kfont_context *ctx, int fd)
 {
 	struct console_font_op cfo;
 
+	if (!is_kd_text(ctx, fd))
+		return -1;
+
 	cfo.op = KD_FONT_OP_SET_DEFAULT;
 
 	if (ioctl(fd, KDFONTOP, &cfo)) {
@@ -60,6 +80,9 @@ get_font_kdfontop(struct kfont_context *ctx, int consolefd,
 {
 	struct console_font_op cfo;
 
+	if (!is_kd_text(ctx, consolefd))
+		return -1;
+
 	cfo.op = KD_FONT_OP_GET;
 	cfo.flags = 0;
 	cfo.width = 32;
@@ -162,6 +185,9 @@ put_font_kdfontop(struct kfont_context *ctx, int consolefd, unsigned char *buf,
 {
 	struct console_font_op cfo;
 
+	if (!is_kd_text(ctx, consolefd))
+		return -1;
+
 	if (vpitch == 32 && width <= 32)
 		cfo.op        = KD_FONT_OP_SET;
 	else {