diff --git a/conf/fail2ban/jail.local b/conf/fail2ban/jail.local
index b9340e52..70725969 100644
--- a/conf/fail2ban/jail.local
+++ b/conf/fail2ban/jail.local
@@ -4,7 +4,7 @@
 # Whitelist our own IP addresses. 127.0.0.1/8 is the default. But our status checks
 # ping services over the public interface so we should whitelist that address of
 # ours too. The string is substituted during installation.
-ignoreip = 127.0.0.1/8 PUBLIC_IP
+ignoreip = 127.0.0.1/8 PUBLIC_IP REMOTE_IP
 
 # JAILS
 
diff --git a/setup/system.sh b/setup/system.sh
index c2fa1b9d..99b7974e 100755
--- a/setup/system.sh
+++ b/setup/system.sh
@@ -282,8 +282,12 @@ restart_service resolvconf
 # ### Fail2Ban Service
 
 # Configure the Fail2Ban installation to prevent dumb bruce-force attacks against dovecot, postfix and ssh
+#
+# We will whitelist our public IP and the IP of the user performing the install
+IP_ADDRESS_OF_USER=$(pinky -w `logname` | tail -n+2 | tail -n1 | awk '{print $(NF)}')
 cat conf/fail2ban/jail.local \
 	| sed "s/PUBLIC_IP/$PUBLIC_IP/g" \
+	| sed "s/REMOTE_IP/$IP_ADDRESS_OF_USER/g" \
 	> /etc/fail2ban/jail.local
 cp conf/fail2ban/dovecotimap.conf /etc/fail2ban/filter.d/dovecotimap.conf