diff --git a/src/Makefile.am b/src/Makefile.am
index b5eba731..ff65ff7f 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -15,6 +15,7 @@ libinput_la_SOURCES =			\
 	evdev-mt-touchpad-tap.c		\
 	evdev-mt-touchpad-buttons.c	\
 	evdev-mt-touchpad-edge-scroll.c	\
+	evdev-mt-touchpad-gestures.c	\
 	filter.c			\
 	filter.h			\
 	filter-private.h		\
diff --git a/src/evdev-mt-touchpad-buttons.c b/src/evdev-mt-touchpad-buttons.c
index 9dbb5137..12f80234 100644
--- a/src/evdev-mt-touchpad-buttons.c
+++ b/src/evdev-mt-touchpad-buttons.c
@@ -156,7 +156,6 @@ tp_button_set_state(struct tp_dispatch *tp, struct tp_touch *t,
 		break;
 	case BUTTON_STATE_AREA:
 		t->button.curr = BUTTON_EVENT_IN_AREA;
-		tp_set_pointer(tp, t);
 		break;
 	case BUTTON_STATE_BOTTOM:
 		t->button.curr = event;
diff --git a/src/evdev-mt-touchpad-edge-scroll.c b/src/evdev-mt-touchpad-edge-scroll.c
index df181d51..28f29c2f 100644
--- a/src/evdev-mt-touchpad-edge-scroll.c
+++ b/src/evdev-mt-touchpad-edge-scroll.c
@@ -86,7 +86,6 @@ tp_edge_scroll_set_state(struct tp_dispatch *tp,
 		break;
 	case EDGE_SCROLL_TOUCH_STATE_AREA:
 		t->scroll.edge = EDGE_NONE;
-		tp_set_pointer(tp, t);
 		break;
 	}
 }
diff --git a/src/evdev-mt-touchpad-gestures.c b/src/evdev-mt-touchpad-gestures.c
new file mode 100644
index 00000000..4317d282
--- /dev/null
+++ b/src/evdev-mt-touchpad-gestures.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright © 2015 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and
+ * its documentation for any purpose is hereby granted without fee, provided
+ * that the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of the copyright holders not be used in
+ * advertising or publicity pertaining to distribution of the software
+ * without specific, written prior permission.  The copyright holders make
+ * no representations about the suitability of this software for any
+ * purpose.  It is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+ * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+ * CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include "config.h"
+
+#include <assert.h>
+#include <math.h>
+#include <stdbool.h>
+#include <limits.h>
+
+#include "evdev-mt-touchpad.h"
+
+#define DEFAULT_GESTURE_SWITCH_TIMEOUT 100 /* ms */
+
+void
+tp_gesture_start(struct tp_dispatch *tp, uint64_t time)
+{
+	if (tp->gesture.started)
+		return;
+
+	switch (tp->gesture.finger_count) {
+	case 2:
+		/* NOP */
+		break;
+	}
+	tp->gesture.started = true;
+}
+
+void
+tp_gesture_post_events(struct tp_dispatch *tp, uint64_t time)
+{
+	if (tp->gesture.finger_count == 0)
+		return;
+
+	/* When tap-and-dragging, or a clickpad is clicked force 1fg mode */
+	if (tp_tap_dragging(tp) || (tp->buttons.is_clickpad && tp->buttons.state)) {
+		tp_gesture_stop(tp, time);
+		tp->gesture.finger_count = 1;
+		tp->gesture.finger_count_pending = 0;
+	}
+
+	/* Don't send events when we're unsure in which mode we are */
+	if (tp->gesture.finger_count_pending)
+		return;
+
+	switch (tp->gesture.finger_count) {
+	case 1:
+		tp_gesture_post_pointer_motion(tp, time);
+		break;
+	case 2:
+		tp_gesture_post_twofinger_scroll(tp, time);
+		break;
+	}
+}
+
+void
+tp_gesture_stop(struct tp_dispatch *tp, uint64_t time)
+{
+	if (!tp->gesture.started)
+		return;
+
+	switch (tp->gesture.finger_count) {
+	case 2:
+		tp_gesture_stop_twofinger_scroll(tp, time);
+		break;
+	}
+	tp->gesture.started = false;
+}
+
+static void
+tp_gesture_finger_count_switch_timeout(uint64_t now, void *data)
+{
+	struct tp_dispatch *tp = data;
+
+	if (!tp->gesture.finger_count_pending)
+		return;
+
+	tp_gesture_stop(tp, now); /* End current gesture */
+	tp->gesture.finger_count = tp->gesture.finger_count_pending;
+	tp->gesture.finger_count_pending = 0;
+}
+
+void
+tp_gesture_handle_state(struct tp_dispatch *tp, uint64_t time)
+{
+	unsigned int active_touches = 0;
+	struct tp_touch *t;
+
+	tp_for_each_touch(tp, t)
+		if (tp_touch_active(tp, t))
+			active_touches++;
+
+	if (active_touches != tp->gesture.finger_count) {
+		/* If all fingers are lifted immediately end the gesture */
+		if (active_touches == 0) {
+			tp_gesture_stop(tp, time);
+			tp->gesture.finger_count = 0;
+			tp->gesture.finger_count_pending = 0;
+		/* Immediately switch to new mode to avoid initial latency */
+		} else if (!tp->gesture.started) {
+			tp->gesture.finger_count = active_touches;
+			tp->gesture.finger_count_pending = 0;
+		/* Else debounce finger changes */
+		} else if (active_touches != tp->gesture.finger_count_pending) {
+			tp->gesture.finger_count_pending = active_touches;
+			libinput_timer_set(&tp->gesture.finger_count_switch_timer,
+				time + DEFAULT_GESTURE_SWITCH_TIMEOUT);
+		}
+	} else {
+		 tp->gesture.finger_count_pending = 0;
+	}
+}
+
+int
+tp_init_gesture(struct tp_dispatch *tp)
+{
+	libinput_timer_init(&tp->gesture.finger_count_switch_timer,
+			    tp->device->base.seat->libinput,
+			    tp_gesture_finger_count_switch_timeout, tp);
+	return 0;
+}
+
+void
+tp_remove_gesture(struct tp_dispatch *tp)
+{
+	libinput_timer_cancel(&tp->gesture.finger_count_switch_timer);
+}
diff --git a/src/evdev-mt-touchpad.c b/src/evdev-mt-touchpad.c
index e281d489..32ed7bf5 100644
--- a/src/evdev-mt-touchpad.c
+++ b/src/evdev-mt-touchpad.c
@@ -227,7 +227,6 @@ tp_end_touch(struct tp_dispatch *tp, struct tp_touch *t, uint64_t time)
 	}
 
 	t->dirty = true;
-	t->is_pointer = false;
 	t->palm.is_palm = false;
 	t->state = TOUCH_END;
 	t->pinned.is_pinned = false;
@@ -424,7 +423,6 @@ tp_unpin_finger(struct tp_dispatch *tp, struct tp_touch *t)
 	if (xdist * xdist + ydist * ydist >=
 			tp->buttons.motion_dist * tp->buttons.motion_dist) {
 		t->pinned.is_pinned = false;
-		tp_set_pointer(tp, t);
 		return;
 	}
 
@@ -439,14 +437,13 @@ tp_pin_fingers(struct tp_dispatch *tp)
 	struct tp_touch *t;
 
 	tp_for_each_touch(tp, t) {
-		t->is_pointer = false;
 		t->pinned.is_pinned = true;
 		t->pinned.center_x = t->x;
 		t->pinned.center_y = t->y;
 	}
 }
 
-static int
+int
 tp_touch_active(struct tp_dispatch *tp, struct tp_touch *t)
 {
 	return (t->state == TOUCH_BEGIN || t->state == TOUCH_UPDATE) &&
@@ -456,21 +453,6 @@ tp_touch_active(struct tp_dispatch *tp, struct tp_touch *t)
 		tp_edge_scroll_touch_active(tp, t);
 }
 
-void
-tp_set_pointer(struct tp_dispatch *tp, struct tp_touch *t)
-{
-	struct tp_touch *tmp = NULL;
-
-	/* Only set the touch as pointer if we don't have one yet */
-	tp_for_each_touch(tp, tmp) {
-		if (tmp->is_pointer)
-			return;
-	}
-
-	if (tp_touch_active(tp, t))
-		t->is_pointer = true;
-}
-
 static void
 tp_palm_detect(struct tp_dispatch *tp, struct tp_touch *t, uint64_t time)
 {
@@ -487,7 +469,6 @@ tp_palm_detect(struct tp_dispatch *tp, struct tp_touch *t, uint64_t time)
 			int dirs = vector_get_direction(t->x - t->palm.x, t->y - t->palm.y);
 			if ((dirs & DIRECTIONS) && !(dirs & ~DIRECTIONS)) {
 				t->palm.is_palm = false;
-				tp_set_pointer(tp, t);
 			}
 		}
 		return;
@@ -530,8 +511,6 @@ tp_get_average_touches_delta(struct tp_dispatch *tp, double *dx, double *dy)
 			*dx += tmpx;
 			*dy += tmpy;
 		}
-		/* Stop spurious MOTION events at the end of scrolling */
-		t->is_pointer = false;
 	}
 
 	if (nchanged == 0)
@@ -541,80 +520,30 @@ tp_get_average_touches_delta(struct tp_dispatch *tp, double *dx, double *dy)
 	*dy /= nchanged;
 }
 
-static void
-tp_post_twofinger_scroll(struct tp_dispatch *tp, uint64_t time)
+void
+tp_gesture_post_twofinger_scroll(struct tp_dispatch *tp, uint64_t time)
 {
 	double dx = 0, dy =0;
 
 	tp_get_average_touches_delta(tp, &dx, &dy);
 	tp_filter_motion(tp, &dx, &dy, NULL, NULL, time);
 
+	if (dx == 0.0 && dy == 0.0)
+		return;	
+
+	tp_gesture_start(tp, time);
 	evdev_post_scroll(tp->device,
 			  time,
 			  LIBINPUT_POINTER_AXIS_SOURCE_FINGER,
 			  dx, dy);
-	tp->scroll.twofinger_state = TWOFINGER_SCROLL_STATE_ACTIVE;
 }
 
-static void
-tp_twofinger_stop_scroll(struct tp_dispatch *tp, uint64_t time)
+void
+tp_gesture_stop_twofinger_scroll(struct tp_dispatch *tp, uint64_t time)
 {
-	struct tp_touch *t, *ptr = NULL;
-	int nfingers_down = 0;
-
 	evdev_stop_scroll(tp->device,
 			  time,
 			  LIBINPUT_POINTER_AXIS_SOURCE_FINGER);
-
-	/* If we were scrolling and now there's exactly 1 active finger,
-	   switch back to pointer movement */
-	if (tp->scroll.twofinger_state == TWOFINGER_SCROLL_STATE_ACTIVE) {
-		tp_for_each_touch(tp, t) {
-			if (tp_touch_active(tp, t)) {
-				nfingers_down++;
-				if (ptr == NULL)
-					ptr = t;
-			}
-		}
-
-		if (nfingers_down == 1)
-			tp_set_pointer(tp, ptr);
-	}
-
-	tp->scroll.twofinger_state = TWOFINGER_SCROLL_STATE_NONE;
-}
-
-static int
-tp_twofinger_scroll_post_events(struct tp_dispatch *tp, uint64_t time)
-{
-	struct tp_touch *t;
-	int nfingers_down = 0;
-
-	if (tp->scroll.method != LIBINPUT_CONFIG_SCROLL_2FG)
-		return 0;
-
-	/* No 2fg scrolling during tap-n-drag */
-	if (tp_tap_dragging(tp))
-		return 0;
-
-	/* No 2fg scrolling while a clickpad is clicked */
-	if (tp->buttons.is_clickpad && tp->buttons.state)
-		return 0;
-
-	/* Only count active touches for 2 finger scrolling */
-	tp_for_each_touch(tp, t) {
-		if (tp_touch_active(tp, t))
-			nfingers_down++;
-	}
-
-	if (nfingers_down == 2) {
-		tp_post_twofinger_scroll(tp, time);
-		return 1;
-	}
-
-	tp_twofinger_stop_scroll(tp, time);
-
-	return 0;
 }
 
 static void
@@ -718,6 +647,8 @@ tp_process_state(struct tp_dispatch *tp, uint64_t time)
 	if ((tp->queued & TOUCHPAD_EVENT_BUTTON_PRESS) &&
 	    tp->buttons.is_clickpad)
 		tp_pin_fingers(tp);
+
+	tp_gesture_handle_state(tp, time);
 }
 
 static void
@@ -748,24 +679,6 @@ tp_post_process_state(struct tp_dispatch *tp, uint64_t time)
 	tp->queued = TOUCHPAD_EVENT_NONE;
 }
 
-static void
-tp_get_pointer_delta(struct tp_dispatch *tp, double *dx, double *dy)
-{
-	struct tp_touch *t = tp_current_touch(tp);
-
-	if (!t->is_pointer) {
-		tp_for_each_touch(tp, t) {
-			if (t->is_pointer)
-				break;
-		}
-	}
-
-	if (!t->is_pointer || !t->dirty)
-		return;
-
-	tp_get_delta(t, dx, dy);
-}
-
 static void
 tp_get_combined_touches_delta(struct tp_dispatch *tp, double *dx, double *dy)
 {
@@ -785,8 +698,8 @@ tp_get_combined_touches_delta(struct tp_dispatch *tp, double *dx, double *dy)
 	}
 }
 
-static void
-tp_post_pointer_motion(struct tp_dispatch *tp, uint64_t time)
+void
+tp_gesture_post_pointer_motion(struct tp_dispatch *tp, uint64_t time)
 {
 	double dx = 0.0, dy = 0.0;
 	double dx_unaccel, dy_unaccel;
@@ -795,7 +708,7 @@ tp_post_pointer_motion(struct tp_dispatch *tp, uint64_t time)
 	if (tp->buttons.is_clickpad && tp->buttons.state)
 		tp_get_combined_touches_delta(tp, &dx, &dy);
 	else
-		tp_get_pointer_delta(tp, &dx, &dy);
+		tp_get_average_touches_delta(tp, &dx, &dy);
 
 	tp_filter_motion(tp, &dx, &dy, &dx_unaccel, &dy_unaccel, time);
 
@@ -821,16 +734,14 @@ tp_post_events(struct tp_dispatch *tp, uint64_t time)
 
 	if (filter_motion || tp->sendevents.trackpoint_active) {
 		tp_edge_scroll_stop_events(tp, time);
-		tp_twofinger_stop_scroll(tp, time);
+		tp_gesture_stop(tp, time);
 		return;
 	}
 
 	if (tp_edge_scroll_post_events(tp, time) != 0)
 		return;
-	if (tp_twofinger_scroll_post_events(tp, time) != 0)
-		return;
 
-	tp_post_pointer_motion(tp, time);
+	tp_gesture_post_events(tp, time);
 }
 
 static void
@@ -887,6 +798,7 @@ tp_remove(struct evdev_dispatch *dispatch)
 	tp_remove_buttons(tp);
 	tp_remove_sendevents(tp);
 	tp_remove_edge_scroll(tp);
+	tp_remove_gesture(tp);
 }
 
 static void
@@ -979,7 +891,7 @@ tp_trackpoint_event(uint64_t time, struct libinput_event *event, void *data)
 
 	if (!tp->sendevents.trackpoint_active) {
 		tp_edge_scroll_stop_events(tp, time);
-		tp_twofinger_stop_scroll(tp, time);
+		tp_gesture_stop(tp, time);
 		tp_tap_suspend(tp, time);
 		tp->sendevents.trackpoint_active = true;
 	}
@@ -1220,7 +1132,7 @@ tp_scroll_config_scroll_method_set_method(struct libinput_device *device,
 		return LIBINPUT_CONFIG_STATUS_SUCCESS;
 
 	tp_edge_scroll_stop_events(tp, time);
-	tp_twofinger_stop_scroll(tp, time);
+	tp_gesture_stop_twofinger_scroll(tp, time);
 
 	tp->scroll.method = method;
 
@@ -1358,6 +1270,9 @@ tp_init(struct tp_dispatch *tp,
 	if (tp_init_scroll(tp, device) != 0)
 		return -1;
 
+	if (tp_init_gesture(tp) != 0)
+		return -1;
+
 	device->seat_caps |= EVDEV_DEVICE_POINTER;
 
 	return 0;
diff --git a/src/evdev-mt-touchpad.h b/src/evdev-mt-touchpad.h
index d552c26d..3e4ac0fd 100644
--- a/src/evdev-mt-touchpad.h
+++ b/src/evdev-mt-touchpad.h
@@ -117,11 +117,6 @@ enum tp_edge_scroll_touch_state {
 	EDGE_SCROLL_TOUCH_STATE_AREA,
 };
 
-enum tp_twofinger_scroll_state {
-	TWOFINGER_SCROLL_STATE_NONE,
-	TWOFINGER_SCROLL_STATE_ACTIVE,
-};
-
 struct tp_motion {
 	int32_t x;
 	int32_t y;
@@ -132,7 +127,6 @@ struct tp_touch {
 	enum touch_state state;
 	bool has_ended;				/* TRACKING_ID == -1 */
 	bool dirty;
-	bool is_pointer;			/* the pointer-controlling touch */
 	int32_t x;
 	int32_t y;
 	uint64_t millis;
@@ -215,6 +209,13 @@ struct tp_dispatch {
 		double y_scale_coeff;
 	} accel;
 
+	struct {
+		bool started;
+		unsigned int finger_count;
+		unsigned int finger_count_pending;
+		struct libinput_timer finger_count_switch_timer;
+	} gesture;
+
 	struct {
 		bool is_clickpad;		/* true for clickpads */
 		bool has_topbuttons;
@@ -252,7 +253,6 @@ struct tp_dispatch {
 		enum libinput_config_scroll_method method;
 		int32_t right_edge;
 		int32_t bottom_edge;
-		enum tp_twofinger_scroll_state twofinger_state;
 	} scroll;
 
 	enum touchpad_event queued;
@@ -286,15 +286,15 @@ struct tp_dispatch {
 void
 tp_get_delta(struct tp_touch *t, double *dx, double *dy);
 
-void
-tp_set_pointer(struct tp_dispatch *tp, struct tp_touch *t);
-
 void
 tp_filter_motion(struct tp_dispatch *tp,
 	         double *dx, double *dy,
 	         double *dx_unaccel, double *dy_unaccel,
 		 uint64_t time);
 
+int
+tp_touch_active(struct tp_dispatch *tp, struct tp_touch *t);
+
 int
 tp_tap_handle_state(struct tp_dispatch *tp, uint64_t time);
 
@@ -367,4 +367,31 @@ tp_edge_scroll_stop_events(struct tp_dispatch *tp, uint64_t time);
 int
 tp_edge_scroll_touch_active(struct tp_dispatch *tp, struct tp_touch *t);
 
+int
+tp_init_gesture(struct tp_dispatch *tp);
+
+void
+tp_remove_gesture(struct tp_dispatch *tp);
+
+void
+tp_gesture_start(struct tp_dispatch *tp, uint64_t time);
+
+void
+tp_gesture_stop(struct tp_dispatch *tp, uint64_t time);
+
+void
+tp_gesture_handle_state(struct tp_dispatch *tp, uint64_t time);
+
+void
+tp_gesture_post_events(struct tp_dispatch *tp, uint64_t time);
+
+void
+tp_gesture_post_twofinger_scroll(struct tp_dispatch *tp, uint64_t time);
+
+void
+tp_gesture_stop_twofinger_scroll(struct tp_dispatch *tp, uint64_t time);
+
+void
+tp_gesture_post_pointer_motion(struct tp_dispatch *tp, uint64_t time);
+
 #endif
diff --git a/test/litest.c b/test/litest.c
index 16d92399..0660dabb 100644
--- a/test/litest.c
+++ b/test/litest.c
@@ -1380,6 +1380,12 @@ litest_timeout_buttonscroll(void)
 	msleep(300);
 }
 
+void
+litest_timeout_finger_switch(void)
+{
+	msleep(120);
+}
+
 void
 litest_push_event_frame(struct litest_device *dev)
 {
diff --git a/test/litest.h b/test/litest.h
index 60ec4587..d78bf0eb 100644
--- a/test/litest.h
+++ b/test/litest.h
@@ -180,6 +180,7 @@ struct libevdev_uinput * litest_create_uinput_abs_device(const char *name,
 void litest_timeout_tap(void);
 void litest_timeout_softbuttons(void);
 void litest_timeout_buttonscroll(void);
+void litest_timeout_finger_switch(void);
 
 void litest_push_event_frame(struct litest_device *dev);
 void litest_pop_event_frame(struct litest_device *dev);
diff --git a/test/touchpad.c b/test/touchpad.c
index 9b3ba812..b70d3732 100644
--- a/test/touchpad.c
+++ b/test/touchpad.c
@@ -1871,6 +1871,9 @@ START_TEST(touchpad_2fg_scroll_return_to_motion)
 	litest_touch_move_to(dev, 0, 47, 50, 47, 70, 5, 0);
 	litest_touch_move_to(dev, 1, 53, 50, 53, 70, 5, 0);
 	litest_touch_up(dev, 1);
+	libinput_dispatch(li);
+	litest_timeout_finger_switch();
+	libinput_dispatch(li);
 	litest_assert_only_typed_events(li, LIBINPUT_EVENT_POINTER_AXIS);
 
 	litest_touch_move_to(dev, 0, 47, 70, 47, 50, 10, 0);
@@ -1881,6 +1884,9 @@ START_TEST(touchpad_2fg_scroll_return_to_motion)
 	litest_touch_move_to(dev, 0, 47, 50, 47, 70, 5, 0);
 	litest_touch_move_to(dev, 1, 53, 50, 53, 70, 5, 0);
 	litest_touch_up(dev, 0);
+	libinput_dispatch(li);
+	litest_timeout_finger_switch();
+	libinput_dispatch(li);
 	litest_assert_only_typed_events(li, LIBINPUT_EVENT_POINTER_AXIS);
 
 	/* move with second finger */