diff --git a/containments/taskpanel/package/contents/ui/TaskSwitcher.qml b/containments/taskpanel/package/contents/ui/TaskSwitcher.qml
index e16859ea..bfabf61b 100644
--- a/containments/taskpanel/package/contents/ui/TaskSwitcher.qml
+++ b/containments/taskpanel/package/contents/ui/TaskSwitcher.qml
@@ -99,6 +99,7 @@ FullScreenPanel {
             script: {
                 if (tasksView.contentY <= -tasksView.headerItem.height || tasksView.contentY >= tasksView.contentHeight - tasksView.headerItem.height) {
                     window.visible = false;
+                    setSingleActiveWindow(currentTaskIndex);
                 }
             }
         }
diff --git a/containments/taskpanel/package/contents/ui/main.qml b/containments/taskpanel/package/contents/ui/main.qml
index 42c60e45..7a3862ed 100644
--- a/containments/taskpanel/package/contents/ui/main.qml
+++ b/containments/taskpanel/package/contents/ui/main.qml
@@ -104,7 +104,6 @@ PlasmaCore.ColorScope {
                 enabled: taskSwitcher.tasksCount > 0;
                 iconSource: "window-list"
                 onClicked: {
-                    taskSwitcher.currentTaskIndex = -1;
                     taskSwitcher.visible ? taskSwitcher.hide() : taskSwitcher.show();
                 }
                 onPressed: mainMouseArea.managePressed(mouse);