diff --git a/config/etc/ssh/ssh_config.d/30-1password-ssh-agent.conf b/config/etc/ssh/ssh_config.d/30-1password-ssh-agent.conf
index d9b3b0c..58d577b 100644
--- a/config/etc/ssh/ssh_config.d/30-1password-ssh-agent.conf
+++ b/config/etc/ssh/ssh_config.d/30-1password-ssh-agent.conf
@@ -1,2 +1,5 @@
-Host *
+Match exec "SSH_AUTH_SOCK=$HOME/.1password/agent.sock ssh-add -l &>/dev/null"
 	IdentityAgent ~/.1password/agent.sock
+
+Match exec "test -n \"$SSH_CONNECTION\""
+	IdentityAgent SSH_AUTH_SOCK