make dns resolver retrying explicit

management/dns_update.py

@@ -1110,35 +1110,33 @@ def set_secondary_dns(hostnames, env):
 	if len(hostnames) > 0:
 		# Validate that all hostnames are valid and that all zone-xfer IP addresses are valid.
 		resolver = dns.resolver.get_default_resolver()
-		resolver.timeout = 3
+		resolver.timeout = 5
-		resolver.lifetime = 3
+		resolver.lifetime = 5
 		for item in hostnames:
 			if not item.startswith("xfr:"):
 				# Resolve hostname.
+				tries = 2
+				
+				while tries > 0:
+					tries = tries - 1
 					try:
 						response = resolver.resolve(item, "A")
+						tries = 0
 					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 						logging.debug('Error on resolving ipv4 address, trying ipv6')
 						try:
 							response = resolver.resolve(item, "AAAA")
+							tries = 0
 						except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 							raise ValueError("Could not resolve the IP address of %s." % item)
 						except (dns.resolver.Timeout):
-					resolver.timeout = 5
+							logging.debug('Timeout on resolving ipv6 address')
-					resolver.lifetime = 5
+							if tries < 1:
-					logging.debug('Timeout on resolving ipv4 address re-trying')
+								raise ValueError("Could not resolve the IP address of %s due to timeout." % item)
-					try:
+					except (dns.resolver.Timeout):
-						response = resolver.resolve(item, "A")
+						logging.debug('Timeout on resolving ipv4 address')
-					except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
+						if tries < 1:
-						logging.debug('Error on resolving ipv4 address, trying ipv6 (2)')
-						try:
-							response = resolver.resolve(item, "AAAA")
-						except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-							raise ValueError("Could not resolve the IP address of %s." % item)
-					except (dns.resolver.Timeout):
 							raise ValueError("Could not resolve the IP address of %s due to timeout." % item)
-					resolver.timeout = 3
-					resolver.lifetime = 3
 			else:
 				# Validate IP address.
 				try:

management/status_checks.py

@@ -801,30 +801,25 @@ def query_dns(qname, rtype, nxdomain='[Not Set]', at=None, as_list=False):
 		resolver.nameservers = [at]
 
 	# Set a timeout so that a non-responsive server doesn't hold us back.
-	resolver.timeout = 3
-	resolver.lifetime = 3
-
-	# Do the query.
-	try:
-		response = resolver.resolve(qname, rtype, search=True)
-	except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
-		# Host did not have an answer for this query; not sure what the
-		# difference is between the two exceptions.
-		logging.debug("No result for dns lookup %s, %s", qname, rtype)
-		return nxdomain
-	except dns.exception.Timeout:
-		logging.debug("Timeout on dns lookup %s, %s. Retrying", qname, rtype)
 	resolver.timeout = 5
 	resolver.lifetime = 5
+
+	tries = 2
+	# Do the query.
+	while tries > 0:
+		tries = tries - 1
 		try:
 			response = resolver.resolve(qname, rtype, search=True)
+			tries = 0
 		except (dns.resolver.NoNameservers, dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
 			# Host did not have an answer for this query; not sure what the
 			# difference is between the two exceptions.
-			logging.debug("No result for dns lookup %s, %s (2)", qname, rtype)
+			logging.debug("No result for dns lookup %s, %s (%d)", qname, rtype, tries)
+			if tries < 1:
 				return nxdomain
 		except dns.exception.Timeout:
-			logging.debug("Timeout on dns lookup %s, %s.", qname, rtype)
+			logging.debug("Timeout on dns lookup %s, %s (%d)", qname, rtype, tries)
+			if tries < 1:
 				return "[timeout]"
 
 	# Normalize IP addresses. IP address --- especially IPv6 addresses --- can