diff --git a/daemon/aux.c b/daemon/aux.c
index cbb645024..459f781a6 100644
--- a/daemon/aux.c
+++ b/daemon/aux.c
@@ -212,3 +212,11 @@ int g_tree_find_all_cmp(void *k, void *v, void *d) {
 		g_queue_push_tail(q, v);
 	return FALSE;
 }
+unsigned int uint32_hash(const void *p) {
+	const u_int32_t *a = p;
+	return *a;
+}
+int uint32_eq(const void *a, const void *b) {
+	const u_int32_t *A = a, *B = b;
+	return (*A == *B) ? TRUE : FALSE;
+}
diff --git a/daemon/aux.h b/daemon/aux.h
index 80a731457..615a4474d 100644
--- a/daemon/aux.h
+++ b/daemon/aux.h
@@ -67,6 +67,8 @@ INLINE void strdupfree(char **, const char *);
 char *get_thread_buf(void);
 unsigned int in6_addr_hash(const void *p);
 int in6_addr_eq(const void *a, const void *b);
+unsigned int uint32_hash(const void *p);
+int uint32_eq(const void *a, const void *b);
 
 
 
diff --git a/daemon/call.c b/daemon/call.c
index a12b42ce4..fbaee346d 100644
--- a/daemon/call.c
+++ b/daemon/call.c
@@ -764,14 +764,16 @@ loop_ok:
 	mutex_lock(&out_srtp->out_lock);
 
 	/* return values are: 0 = forward packet, -1 = error/dont forward,
-	 * 1 = forward and push update to redis */
+	 * 1 = forward and push update to redis and kernel */
 	if (rwf_in)
 		handler_ret = rwf_in(s, in_srtp);
 	if (handler_ret >= 0 && rwf_out)
 		handler_ret += rwf_out(s, out_srtp);
 
-	if (handler_ret > 0)
+	if (handler_ret > 0) {
+		__unkernelize(stream);
 		update = 1;
+	}
 
 	mutex_unlock(&out_srtp->out_lock);
 	mutex_unlock(&in_srtp->in_lock);
@@ -860,12 +862,6 @@ kernel_check:
 	if (PS_ISSET(stream, NO_KERNEL_SUPPORT))
 		goto forward;
 
-	if (sink && sink->crypto.ssrc_mismatch && !stun_ret) {
-		ilog(LOG_INFO, "SSRC changed, unkernelizing media stream");
-		__unkernelize(stream);
-		goto forward;
-	}
-
 	if (PS_ISSET(stream, CONFIRMED) && sink && PS_ARESET2(sink, CONFIRMED, FILLED))
 		kernelize(stream);
 
@@ -1384,7 +1380,6 @@ static void callmaster_timer(void *ptr) {
 	struct stream_fd *sfd;
 	struct rtp_stats *rs;
 	unsigned int pt;
-	struct rtp_ssrc_entry *cur_ssrc;
 
 	ZERO(hlp);
 
@@ -1451,11 +1446,6 @@ static void callmaster_timer(void *ptr) {
 			mutex_lock(&sink->out_lock);
 			if (sink->crypto.params.crypto_suite
 					&& ke->target.encrypt.last_index - sink->crypto.last_index > 0x4000) {
-				// Keep the SSRC list in sync too.
-			  	cur_ssrc = find_ssrc(ke->target.ssrc, sink->crypto.ssrc_list);
-				if (cur_ssrc)
-					cur_ssrc->index = ke->target.encrypt.last_index;
-
 				sink->crypto.last_index = ke->target.encrypt.last_index;
 				update = 1;
 			}
@@ -2943,7 +2933,6 @@ void call_destroy(struct call *c) {
 		dtls_shutdown(ps);
 		ps->sfd = NULL;
 		crypto_cleanup(&ps->crypto);
-		free_ssrc_list(ps->crypto.ssrc_list);
 
 		ps->rtp_sink = NULL;
 		ps->rtcp_sink = NULL;
diff --git a/daemon/crypto.c b/daemon/crypto.c
index b069c112b..41f8f7ba8 100644
--- a/daemon/crypto.c
+++ b/daemon/crypto.c
@@ -573,35 +573,31 @@ void crypto_dump_keys(struct crypto_context *in, struct crypto_context *out) {
 	dump_key(out);
 }
 
-struct rtp_ssrc_entry *find_ssrc(u_int32_t ssrc, struct rtp_ssrc_entry *list) {
-	for (; list; list = list->next) {
-		if (list->ssrc == ssrc)
-			return list;
-	}
-	return 0;
+struct rtp_ssrc_entry *find_ssrc(u_int32_t ssrc, GHashTable *ht) {
+	return g_hash_table_lookup(ht, &ssrc);
 }
 
-void add_ssrc_entry(struct rtp_ssrc_entry *ent, struct rtp_ssrc_entry *list) {
-	struct rtp_ssrc_entry *cur;
-	for (cur = list; list; list = list->next)
-		cur = list;
-	cur->next = ent;
+void add_ssrc_entry(struct rtp_ssrc_entry *ent, GHashTable *ht) {
+	g_hash_table_insert(ht, &ent->ssrc, ent);
 }
 
 struct rtp_ssrc_entry *create_ssrc_entry(u_int32_t ssrc, u_int64_t index) {
 	struct rtp_ssrc_entry *ent;
-	ent = malloc(sizeof(struct rtp_ssrc_entry));
+	ent = g_slice_alloc(sizeof(struct rtp_ssrc_entry));
 	ent->ssrc = ssrc;
 	ent->index = index;
-	ent->next = 0;
 	return ent;
 }
 
-void free_ssrc_list(struct rtp_ssrc_entry *list) {
-	struct rtp_ssrc_entry *tmp;
-	for (tmp = list; tmp; ) {
-		tmp = list->next;
-		free(list);
-		list = tmp;
-	}
+void free_ssrc_table(GHashTable **ht) {
+	if (!*ht)
+		return;
+	g_hash_table_destroy(*ht);
+	*ht = NULL;
+}
+static void free_ssrc_entry(void *p) {
+	g_slice_free1(sizeof(struct rtp_ssrc_entry), p);
+}
+GHashTable *create_ssrc_table(void) {
+	return g_hash_table_new_full(uint32_hash, uint32_eq, free_ssrc_entry, NULL);
 }
diff --git a/daemon/crypto.h b/daemon/crypto.h
index 04b28b66d..b0211bca1 100644
--- a/daemon/crypto.h
+++ b/daemon/crypto.h
@@ -86,17 +86,14 @@ struct crypto_context {
 	/* <from, to>? */
 
 	void *session_key_ctx[2];
+	GHashTable *ssrc_hash;
 
 	int have_session_key:1;
-
-	struct rtp_ssrc_entry   *ssrc_list;
-	int                     ssrc_mismatch;
 };
 
 struct rtp_ssrc_entry {
 	u_int32_t ssrc;
 	u_int64_t index;
-	struct rtp_ssrc_entry *next;
 };
 
 extern const struct crypto_suite crypto_suites[];
@@ -108,10 +105,11 @@ const struct crypto_suite *crypto_find_suite(const str *);
 int crypto_gen_session_key(struct crypto_context *, str *, unsigned char, int);
 void crypto_dump_keys(struct crypto_context *in, struct crypto_context *out);
 
-struct rtp_ssrc_entry *find_ssrc(u_int32_t, struct rtp_ssrc_entry *);
-void add_ssrc_entry(struct rtp_ssrc_entry *, struct rtp_ssrc_entry *);
+struct rtp_ssrc_entry *find_ssrc(u_int32_t, GHashTable *);
+void add_ssrc_entry(struct rtp_ssrc_entry *, GHashTable *);
 struct rtp_ssrc_entry *create_ssrc_entry(u_int32_t, u_int64_t);
-void free_ssrc_list(struct rtp_ssrc_entry *);
+void free_ssrc_table(GHashTable **);
+GHashTable *create_ssrc_table(void);
 
 
 INLINE int crypto_encrypt_rtp(struct crypto_context *c, struct rtp_header *rtp,
@@ -145,6 +143,7 @@ INLINE void crypto_params_cleanup(struct crypto_params *p) {
 }
 INLINE void crypto_cleanup(struct crypto_context *c) {
 	crypto_params_cleanup(&c->params);
+	free_ssrc_table(&c->ssrc_hash);
 	if (!c->params.crypto_suite)
 		return;
 	if (c->params.crypto_suite->session_key_cleanup)
diff --git a/daemon/rtp.c b/daemon/rtp.c
index 00781428a..3b27ce915 100644
--- a/daemon/rtp.c
+++ b/daemon/rtp.c
@@ -180,41 +180,54 @@ void rtp_append_mki(str *s, struct crypto_context *c) {
 int rtp_avp2savp(str *s, struct crypto_context *c) {
 	struct rtp_header *rtp;
 	str payload, to_auth;
+	u_int64_t index;
 	struct rtp_ssrc_entry *cur_ssrc;
+	int update_kernel = 0;
 
 	if (rtp_payload(&rtp, &payload, s))
 		return -1;
 	if (check_session_keys(c))
 		return -1;
 
-	if (G_UNLIKELY(!c->ssrc_list))
-		c->ssrc_list = create_ssrc_entry(rtp->ssrc, ntohs(rtp->seq_num));
+	/* check last known SSRC */
+	if (G_LIKELY(rtp->ssrc == c->ssrc))
+		goto ssrc_ok;
+	if (!c->ssrc) {
+		c->ssrc = rtp->ssrc;
+		update_kernel = 1;
+		goto ssrc_ok;
+	}
+
+	/* SSRC mismatch. stash away last know info */
+	ilog(LOG_DEBUG, "SSRC changed, updating SRTP crypto contexts");
+	if (G_UNLIKELY(!c->ssrc_hash))
+		c->ssrc_hash = create_ssrc_table();
 
-	// Find the entry for the current SSRC.
-	cur_ssrc = find_ssrc(rtp->ssrc, c->ssrc_list);
+	// Find the entry for the last SSRC.
+	cur_ssrc = find_ssrc(c->ssrc, c->ssrc_hash);
 	// If it doesn't exist, create a new entry.
 	if (G_UNLIKELY(!cur_ssrc)) {
-		cur_ssrc = create_ssrc_entry(rtp->ssrc, ntohs(rtp->seq_num));
-		add_ssrc_entry(cur_ssrc, c->ssrc_list);
+		cur_ssrc = create_ssrc_entry(c->ssrc, c->last_index);
+		add_ssrc_entry(cur_ssrc, c->ssrc_hash);
 	}
+	else
+		cur_ssrc->index = c->last_index;
 
 	// New SSRC, set the crypto context.
-	if (G_UNLIKELY(!c->ssrc))
-		c->ssrc = rtp->ssrc;
-	// SSRC has changed.
-	else if (G_UNLIKELY(c->ssrc != rtp->ssrc)) {
-		// Signal stream_packet() to unkernelize.
-		c->ssrc_mismatch = 1;
-		c->ssrc = rtp->ssrc;
+	c->ssrc = rtp->ssrc;
+	cur_ssrc = find_ssrc(rtp->ssrc, c->ssrc_hash);
+	if (G_UNLIKELY(!cur_ssrc))
+		c->last_index = 0;
+	else
 		c->last_index = cur_ssrc->index;
-	} else {
-		c->ssrc_mismatch = 0;
-	}
 
-	cur_ssrc->index = packet_index(c, rtp);
+	update_kernel = 1;
+
+ssrc_ok:
+	index = packet_index(c, rtp);
 
 	/* rfc 3711 section 3.1 */
-	if (!c->params.session_params.unencrypted_srtp && crypto_encrypt_rtp(c, rtp, &payload, cur_ssrc->index))
+	if (!c->params.session_params.unencrypted_srtp && crypto_encrypt_rtp(c, rtp, &payload, index))
 		return -1;
 
 	to_auth = *s;
@@ -222,11 +235,11 @@ int rtp_avp2savp(str *s, struct crypto_context *c) {
 	rtp_append_mki(s, c);
 
 	if (!c->params.session_params.unauthenticated_srtp && c->params.crypto_suite->srtp_auth_tag) {
-		c->params.crypto_suite->hash_rtp(c, s->s + s->len, &to_auth, cur_ssrc->index);
+		c->params.crypto_suite->hash_rtp(c, s->s + s->len, &to_auth, index);
 		s->len += c->params.crypto_suite->srtp_auth_tag;
 	}
 
-	return 0;
+	return update_kernel ? 1 : 0;
 }
 
 /* rfc 3711, section 3.3 */
diff --git a/kernel-module/xt_RTPENGINE.c b/kernel-module/xt_RTPENGINE.c
index d8baaa4c5..b67688307 100644
--- a/kernel-module/xt_RTPENGINE.c
+++ b/kernel-module/xt_RTPENGINE.c
@@ -141,7 +141,6 @@ struct re_crypto_context {
 	struct crypto_shash		*shash;
 	const struct re_cipher		*cipher;
 	const struct re_hmac		*hmac;
-        u_int32_t                       ssrc;
 };
 
 struct rtpengine_stats_a {
@@ -1894,12 +1893,6 @@ static u_int64_t packet_index(struct re_crypto_context *c,
 	u_int32_t roc;
 	u_int32_t v;
 
-	// Keep the crypt context SSRC in sync.
-	if (unlikely(!c->ssrc))
-		c->ssrc = rtp->ssrc;
-	else if (unlikely(c->ssrc != rtp->ssrc))
-		c->ssrc = rtp->ssrc;
-
 	seq = ntohs(rtp->seq_num);
 
 	spin_lock_irqsave(&c->lock, flags);
@@ -2278,7 +2271,7 @@ src_check_ok:
 	rtp_pt_idx = rtp_payload_type(rtp.header, &g->target);
 
 	// Pass to userspace if SSRC has changed.
-	if ((g->encrypt.ssrc) && (g->encrypt.ssrc != rtp.header->ssrc))
+	if (unlikely((g->target.ssrc) && (g->target.ssrc != rtp.header->ssrc)))
 		goto skip_error;
 
 	pkt_idx = packet_index(&g->decrypt, &g->target.decrypt, rtp.header);