From 395f0b157d628b4f073a36a206ebdbdf374f6d05 Mon Sep 17 00:00:00 2001
From: Eduardo Habkost <ehabkost@redhat.com>
Date: Fri, 30 Jan 2009 17:30:27 -0200
Subject: [PATCH 03/54] qemu: vnc: use libgcrypt for DES encryption on authentication

d3des.c has a hack to reverse the bits on each key byte when setting the
key. For using a standard DES implementation, we need to reverse the
bits ourselves.

Signed-off-by: Eduardo Habkost <ehabkost@redhat.com>
---
 qemu/Makefile |    4 +-
 qemu/vnc.c    |   59 +++++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/qemu/Makefile b/qemu/Makefile
index a16ff89..8293bf9 100644
--- a/qemu/Makefile
+++ b/qemu/Makefile
@@ -145,7 +145,7 @@ endif
 ifdef CONFIG_CURSES
 OBJS+=curses.o
 endif
-OBJS+=vnc.o d3des.o
+OBJS+=vnc.o
 
 ifdef CONFIG_COCOA
 OBJS+=cocoa.o
@@ -173,7 +173,7 @@ cocoa.o: cocoa.m
 sdl.o: sdl.c keymaps.c sdl_keysym.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(SDL_CFLAGS) -c -o $@ $<
 
-vnc.o: vnc.c keymaps.c sdl_keysym.h vnchextile.h d3des.c d3des.h
+vnc.o: vnc.c keymaps.c sdl_keysym.h vnchextile.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) $(CONFIG_VNC_TLS_CFLAGS) -c -o $@ $<
 
 curses.o: curses.c keymaps.c curses_keys.h
diff --git a/qemu/vnc.c b/qemu/vnc.c
index 575fd68..41261d4 100644
--- a/qemu/vnc.c
+++ b/qemu/vnc.c
@@ -34,7 +34,7 @@
 
 #include "vnc_keysym.h"
 #include "keymaps.c"
-#include "d3des.h"
+#include <gcrypt.h>
 
 #ifdef CONFIG_VNC_TLS
 #include <gnutls/gnutls.h>
@@ -177,6 +177,8 @@ struct VncState
     size_t read_handler_expect;
     /* input */
     uint8_t modifiers_state[256];
+
+    gcry_cipher_hd_t des_cipher;
 };
 
 static VncState *vnc_state; /* needed for info vnc */
@@ -1655,11 +1657,21 @@ static void make_challenge(VncState *vs)
         vs->challenge[i] = (int) (256.0*rand()/(RAND_MAX+1.0));
 }
 
+static unsigned char bitrev(unsigned char b)
+{
+	unsigned char r = b;
+	r = (r & 0xf0)>>4 | (r & 0x0f)<<4;
+	r = (r & 0xcc)>>2 | (r & 0x33)<<2;
+	r = (r & 0xaa)>>1 | (r & 0x55)<<1;
+	return r;
+}
+
 static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len)
 {
     unsigned char response[VNC_AUTH_CHALLENGE_SIZE];
     int i, j, pwlen;
-    unsigned char key[8];
+    char key[8];
+    gcry_error_t e;
 
     if (!vs->password || !vs->password[0]) {
 	VNC_DEBUG("No password configured on server");
@@ -1674,15 +1686,28 @@ static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len)
 	return 0;
     }
 
+    if (gcry_cipher_reset(vs->des_cipher)) {
+        VNC_DEBUG("f1\n");
+        goto fail;
+    }
+
     memcpy(response, vs->challenge, VNC_AUTH_CHALLENGE_SIZE);
 
     /* Calculate the expected challenge response */
     pwlen = strlen(vs->password);
     for (i=0; i<sizeof(key); i++)
-        key[i] = i<pwlen ? vs->password[i] : 0;
-    deskey(key, EN0);
-    for (j = 0; j < VNC_AUTH_CHALLENGE_SIZE; j += 8)
-        des(response+j, response+j);
+        key[i] = i<pwlen ? bitrev(vs->password[i]) : 0;
+
+    if (gcry_cipher_setkey(vs->des_cipher, key, sizeof(key))) {
+        VNC_DEBUG("f2\n");
+        goto fail;
+    }
+    for (j = 0; j < VNC_AUTH_CHALLENGE_SIZE; j += 8) {
+        if ( (e = gcry_cipher_encrypt(vs->des_cipher, response+j, 8, NULL, 0)) ) {
+            VNC_DEBUG("f3: %s\n", gcry_strerror(e));
+            goto fail;
+        }
+    }
 
     /* Compare expected vs actual challenge response */
     if (memcmp(response, data, VNC_AUTH_CHALLENGE_SIZE) != 0) {
@@ -1703,6 +1728,18 @@ static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len)
 	vnc_read_when(vs, protocol_client_init, 1);
     }
     return 0;
+
+fail:
+    /* Should never happen, but just in case... */
+    vnc_write_u32(vs, 1); /* Reject auth */
+    if (vs->minor >= 8) {
+        static const char err[] = "Internal crypto library error";
+        vnc_write_u32(vs, sizeof(err));
+        vnc_write(vs, err, sizeof(err));
+    }
+    vnc_flush(vs);
+    vnc_client_error(vs);
+    return 0;
 }
 
 static int start_auth_vnc(VncState *vs)
@@ -2310,6 +2347,16 @@ void vnc_display_init(DisplayState *ds)
     vs->as.nchannels = 2;
     vs->as.fmt = AUD_FMT_S16;
     vs->as.endianness = 0;
+
+    if (!gcry_check_version (GCRYPT_VERSION)) {
+        fprintf(stderr, "libgcrypt initialization error\n");
+        exit(1);
+    }
+
+    if (gcry_cipher_open(&vs->des_cipher, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0)) {
+        fprintf(stderr, "libgcrypt DES cipher initialization error\n");
+        exit(1);
+    }
 }
 
 #ifdef CONFIG_VNC_TLS
-- 
1.6.1