From 395f0b157d628b4f073a36a206ebdbdf374f6d05 Mon Sep 17 00:00:00 2001 From: Eduardo Habkost <ehabkost@redhat.com> Date: Fri, 30 Jan 2009 17:30:27 -0200 Subject: [PATCH 03/54] qemu: vnc: use libgcrypt for DES encryption on authentication d3des.c has a hack to reverse the bits on each key byte when setting the key. For using a standard DES implementation, we need to reverse the bits ourselves. Signed-off-by: Eduardo Habkost <ehabkost@redhat.com> --- qemu/Makefile | 4 +- qemu/vnc.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/qemu/Makefile b/qemu/Makefile index a16ff89..8293bf9 100644 --- a/qemu/Makefile +++ b/qemu/Makefile @@ -145,7 +145,7 @@ endif ifdef CONFIG_CURSES OBJS+=curses.o endif -OBJS+=vnc.o d3des.o +OBJS+=vnc.o ifdef CONFIG_COCOA OBJS+=cocoa.o @@ -173,7 +173,7 @@ cocoa.o: cocoa.m sdl.o: sdl.c keymaps.c sdl_keysym.h $(CC) $(CFLAGS) $(CPPFLAGS) $(SDL_CFLAGS) -c -o $@ $< -vnc.o: vnc.c keymaps.c sdl_keysym.h vnchextile.h d3des.c d3des.h +vnc.o: vnc.c keymaps.c sdl_keysym.h vnchextile.h $(CC) $(CFLAGS) $(CPPFLAGS) $(CONFIG_VNC_TLS_CFLAGS) -c -o $@ $< curses.o: curses.c keymaps.c curses_keys.h diff --git a/qemu/vnc.c b/qemu/vnc.c index 575fd68..41261d4 100644 --- a/qemu/vnc.c +++ b/qemu/vnc.c @@ -34,7 +34,7 @@ #include "vnc_keysym.h" #include "keymaps.c" -#include "d3des.h" +#include <gcrypt.h> #ifdef CONFIG_VNC_TLS #include <gnutls/gnutls.h> @@ -177,6 +177,8 @@ struct VncState size_t read_handler_expect; /* input */ uint8_t modifiers_state[256]; + + gcry_cipher_hd_t des_cipher; }; static VncState *vnc_state; /* needed for info vnc */ @@ -1655,11 +1657,21 @@ static void make_challenge(VncState *vs) vs->challenge[i] = (int) (256.0*rand()/(RAND_MAX+1.0)); } +static unsigned char bitrev(unsigned char b) +{ + unsigned char r = b; + r = (r & 0xf0)>>4 | (r & 0x0f)<<4; + r = (r & 0xcc)>>2 | (r & 0x33)<<2; + r = (r & 0xaa)>>1 | (r & 0x55)<<1; + return r; +} + static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len) { unsigned char response[VNC_AUTH_CHALLENGE_SIZE]; int i, j, pwlen; - unsigned char key[8]; + char key[8]; + gcry_error_t e; if (!vs->password || !vs->password[0]) { VNC_DEBUG("No password configured on server"); @@ -1674,15 +1686,28 @@ static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len) return 0; } + if (gcry_cipher_reset(vs->des_cipher)) { + VNC_DEBUG("f1\n"); + goto fail; + } + memcpy(response, vs->challenge, VNC_AUTH_CHALLENGE_SIZE); /* Calculate the expected challenge response */ pwlen = strlen(vs->password); for (i=0; i<sizeof(key); i++) - key[i] = i<pwlen ? vs->password[i] : 0; - deskey(key, EN0); - for (j = 0; j < VNC_AUTH_CHALLENGE_SIZE; j += 8) - des(response+j, response+j); + key[i] = i<pwlen ? bitrev(vs->password[i]) : 0; + + if (gcry_cipher_setkey(vs->des_cipher, key, sizeof(key))) { + VNC_DEBUG("f2\n"); + goto fail; + } + for (j = 0; j < VNC_AUTH_CHALLENGE_SIZE; j += 8) { + if ( (e = gcry_cipher_encrypt(vs->des_cipher, response+j, 8, NULL, 0)) ) { + VNC_DEBUG("f3: %s\n", gcry_strerror(e)); + goto fail; + } + } /* Compare expected vs actual challenge response */ if (memcmp(response, data, VNC_AUTH_CHALLENGE_SIZE) != 0) { @@ -1703,6 +1728,18 @@ static int protocol_client_auth_vnc(VncState *vs, uint8_t *data, size_t len) vnc_read_when(vs, protocol_client_init, 1); } return 0; + +fail: + /* Should never happen, but just in case... */ + vnc_write_u32(vs, 1); /* Reject auth */ + if (vs->minor >= 8) { + static const char err[] = "Internal crypto library error"; + vnc_write_u32(vs, sizeof(err)); + vnc_write(vs, err, sizeof(err)); + } + vnc_flush(vs); + vnc_client_error(vs); + return 0; } static int start_auth_vnc(VncState *vs) @@ -2310,6 +2347,16 @@ void vnc_display_init(DisplayState *ds) vs->as.nchannels = 2; vs->as.fmt = AUD_FMT_S16; vs->as.endianness = 0; + + if (!gcry_check_version (GCRYPT_VERSION)) { + fprintf(stderr, "libgcrypt initialization error\n"); + exit(1); + } + + if (gcry_cipher_open(&vs->des_cipher, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB, 0)) { + fprintf(stderr, "libgcrypt DES cipher initialization error\n"); + exit(1); + } } #ifdef CONFIG_VNC_TLS -- 1.6.1