diff --git a/src/event.rs b/src/event.rs
index de12d3b..19f1ef3 100644
--- a/src/event.rs
+++ b/src/event.rs
@@ -14,7 +14,7 @@
 use cli::Options;
 use display::OnResize;
 use index::{Line, Column, Side, Point};
-use input::{self, MouseBinding, KeyBinding, ActionContext as InputActionContext};
+use input::{self, MouseBinding, KeyBinding};
 use selection::Selection;
 use sync::FairMutex;
 use term::{Term, SizeInfo, TermMode};
@@ -76,28 +76,26 @@
     }
 
     fn update_selection(&mut self, point: Point, side: Side) {
-        self.selection_modified = true;
         // Update selection if one exists
         if let Some(ref mut selection) = *self.selection {
+            self.selection_modified = true;
             selection.update(point, side);
-            return;
         }
-
-        // Otherwise, start a regular selection
-        self.simple_selection(point, side);
     }
 
     fn simple_selection(&mut self, point: Point, side: Side) {
-        self.start_selection(Selection::simple(point, side));
+        *self.selection = Some(Selection::simple(point, side));
+        self.selection_modified = true;
     }
 
     fn semantic_selection(&mut self, point: Point) {
-        let selection = Selection::semantic(point, self.terminal);
-        self.start_selection(selection);
+        *self.selection = Some(Selection::semantic(point, self.terminal));
+        self.selection_modified = true;
     }
 
     fn line_selection(&mut self, point: Point) {
-        self.start_selection(Selection::lines(point));
+        *self.selection = Some(Selection::lines(point));
+        self.selection_modified = true;
     }
 
     fn mouse_coords(&self) -> Option<Point> {
@@ -138,18 +136,6 @@
     }
 }
 
-impl<'a, N: Notify + 'a> ActionContext<'a, N> {
-    /// Helper to create selections
-    fn start_selection(&mut self, selection: Selection) {
-        // Reset beginning of selection once selection is started
-        self.mouse_mut().selection_start_point = None;
-        self.mouse_mut().selection_start_side = None;
-
-        *self.selection = Some(selection);
-        self.selection_modified = true;
-    }
-}
-
 pub enum ClickState {
     None,
     Click,
@@ -171,8 +157,6 @@
     pub column: Column,
     pub cell_side: Side,
     pub lines_scrolled: f32,
-    pub selection_start_point: Option<Point>,
-    pub selection_start_side: Option<Side>,
 }
 
 impl Default for Mouse {
@@ -190,8 +174,6 @@
             column: Column(0),
             cell_side: Side::Left,
             lines_scrolled: 0.0,
-            selection_start_point: None,
-            selection_start_side: None,
         }
     }
 }
diff --git a/src/input.rs b/src/input.rs
index fe897b2..4e78ae1 100644
--- a/src/input.rs
+++ b/src/input.rs
@@ -270,9 +270,11 @@
         self.ctx.mouse_mut().x = x;
         self.ctx.mouse_mut().y = y;
 
-        if let Some((point, cell_side)) = self.get_mouse_pos() {
+        let size_info = self.ctx.size_info();
+        if let Some(point) = size_info.pixels_to_coords(x as usize, y as usize) {
             let prev_line = mem::replace(&mut self.ctx.mouse_mut().line, point.line);
             let prev_col = mem::replace(&mut self.ctx.mouse_mut().column, point.col);
+            let cell_side = self.get_mouse_side();
 
             self.ctx.mouse_mut().cell_side = cell_side;
 
@@ -283,16 +285,10 @@
                     || !self.ctx.terminal_mode().intersects(TermMode::MOUSE_REPORT_CLICK | motion_mode)
                 )
             {
-                let start_point = self.ctx.mouse().selection_start_point;
-                let start_side = self.ctx.mouse().selection_start_side;
-                if let (Some(point), Some(side)) = (start_point, start_side) {
-                    self.ctx.update_selection(point, side);
-                } else {
-                    self.ctx.update_selection(Point {
-                        line: point.line,
-                        col: point.col
-                    }, cell_side);
-                }
+                self.ctx.update_selection(Point {
+                    line: point.line,
+                    col: point.col
+                }, cell_side);
             } else if self.ctx.terminal_mode().intersects(motion_mode)
                 // Only report motion when changing cells
                 && (
@@ -391,13 +387,14 @@
                 ClickState::TripleClick
             },
             _ => {
+                self.ctx.clear_selection();
+
                 // Store click position for accurate selection
-                if let Some((point, side)) = self.get_mouse_pos() {
-                    self.ctx.mouse_mut().selection_start_point = Some(point);
-                    self.ctx.mouse_mut().selection_start_side = Some(side);
+                if let Some(point) = self.ctx.mouse_coords() {
+                    let side = self.get_mouse_side();
+                    self.ctx.simple_selection(point, side);
                 }
 
-                self.ctx.clear_selection();
                 let report_modes = TermMode::MOUSE_REPORT_CLICK | TermMode::MOUSE_DRAG | TermMode::MOUSE_MOTION;
                 if !modifiers.shift && self.ctx.terminal_mode().intersects(report_modes) {
                     match button {
@@ -415,25 +412,20 @@
         };
     }
 
-    fn get_mouse_pos(&self) -> Option<(Point, Side)> {
-        let x = self.ctx.mouse().x;
-        let y = self.ctx.mouse().y;
-
+    fn get_mouse_side(&self) -> Side {
         let size_info = self.ctx.size_info();
-        if let Some(point) = size_info.pixels_to_coords(x as usize, y as usize) {
-            let cell_x = (x as usize - size_info.padding_x as usize) % size_info.cell_width as usize;
-            let half_cell_width = (size_info.cell_width / 2.0) as usize;
+        let x = self.ctx.mouse().x;
 
-            let cell_side = if cell_x > half_cell_width {
-                Side::Right
-            } else {
-                Side::Left
-            };
+        let cell_x = (x as usize - size_info.padding_x as usize) % size_info.cell_width as usize;
+        let half_cell_width = (size_info.cell_width / 2.0) as usize;
 
-            Some((point, cell_side))
+        let cell_side = if cell_x > half_cell_width {
+            Side::Right
         } else {
-            None
-        }
+            Side::Left
+        };
+
+        cell_side
     }
 
     pub fn on_mouse_release(&mut self, button: MouseButton, modifiers: ModifiersState) {