diff --git a/containments/taskpanel/package/contents/ui/main.qml b/containments/taskpanel/package/contents/ui/main.qml
index bed81535..ca8bb5a5 100644
--- a/containments/taskpanel/package/contents/ui/main.qml
+++ b/containments/taskpanel/package/contents/ui/main.qml
@@ -70,7 +70,6 @@ PlasmaCore.ColorScope {
             if (taskSwitcher.visibility == Window.Hidden && taskSwitcher.offset > -taskSwitcher.height + units.gridUnit && taskSwitcher.tasksCount) {
                 taskSwitcher.visible = true;
             }
-            plasmoid.nativeInterface.showDesktop = true;
         }
         onReleased: {
             if (!isDragging) {
diff --git a/containments/taskpanel/taskpanel.cpp b/containments/taskpanel/taskpanel.cpp
index 777e5920..dbf7cc33 100644
--- a/containments/taskpanel/taskpanel.cpp
+++ b/containments/taskpanel/taskpanel.cpp
@@ -119,15 +119,28 @@ QWindow *TaskPanel::panel()
 
 void TaskPanel::setPanel(QWindow *panel)
 {
-    using namespace KWayland::Client;
     if (panel == m_panel) {
         return;
     }
 
+    if (m_panel) {
+        disconnect(m_panel, &QWindow::visibilityChanged, this, &TaskPanel::updatePanelVisibility);
+    }
     m_panel = panel;
+    connect(m_panel, &QWindow::visibilityChanged, this, &TaskPanel::updatePanelVisibility, Qt::QueuedConnection);
     emit panelChanged();
+    updatePanelVisibility();
+}
+
+void TaskPanel::updatePanelVisibility()
+{
+    using namespace KWayland::Client;
+    if (!m_panel->isVisible()) {
+        return;
+    }
+
+    Surface *s = Surface::fromWindow(m_panel);
 
-    Surface *s = Surface::fromWindow(panel);
     if (!s) {
         return;
     }
diff --git a/containments/taskpanel/taskpanel.h b/containments/taskpanel/taskpanel.h
index c2327349..d0d84cc5 100644
--- a/containments/taskpanel/taskpanel.h
+++ b/containments/taskpanel/taskpanel.h
@@ -72,6 +72,7 @@ Q_SIGNALS:
 private:
     void initWayland();
     void updateActiveWindow();
+    void updatePanelVisibility();
     bool m_showingDesktop;
     QWindow *m_panel = nullptr;
     KWayland::Client::PlasmaShellSurface *m_shellSurface = nullptr;