Port boto to boto3 and fix asyncio issue in the management daemon (#2156)

Co-authored-by: Steve Hay <hay.steve@gmail.com>

management/backup.py

@@ -446,25 +446,13 @@ def list_target_files(config):
 			raise ValueError("Connection to rsync host failed: {}".format(reason))
 
 	elif target.scheme == "s3":
-		# match to a Region
+		import boto3.s3
-		import boto.s3
+		from botocore.exceptions import ClientError
-		from boto.exception import BotoServerError
-		custom_region = False
-		for region in boto.s3.regions():
-			if region.endpoint == target.hostname:
-				break
-		else:
-			# If region is not found this is a custom region
-			custom_region = True
 		
+		# separate bucket from path in target
 		bucket = target.path[1:].split('/')[0]
 		path = '/'.join(target.path[1:].split('/')[1:]) + '/'
 
-		# Create a custom region with custom endpoint
-		if custom_region:
-			from boto.s3.connection import S3Connection
-			region = boto.s3.S3RegionInfo(name=bucket, endpoint=target.hostname, connection_cls=S3Connection)
-
 		# If no prefix is specified, set the path to '', otherwise boto won't list the files
 		if path == '/':
 			path = ''
@@ -474,18 +462,15 @@ def list_target_files(config):
 
 		# connect to the region & bucket
 		try:
-			conn = region.connect(aws_access_key_id=config["target_user"], aws_secret_access_key=config["target_pass"])
+			s3 = boto3.client('s3', \
-			bucket = conn.get_bucket(bucket)
+				endpoint_url=f'https://{target.hostname}', \
-		except BotoServerError as e:
+				aws_access_key_id=config['target_user'], \
-			if e.status == 403:
+				aws_secret_access_key=config['target_pass'])
-				raise ValueError("Invalid S3 access key or secret access key.")
+			bucket_objects = s3.list_objects_v2(Bucket=bucket, Prefix=path)['Contents']
-			elif e.status == 404:
+			backup_list = [(key['Key'][len(path):], key['Size']) for key in bucket_objects]
-				raise ValueError("Invalid S3 bucket name.")
+		except ClientError as e:
-			elif e.status == 301:
+			raise ValueError(e)
-				raise ValueError("Incorrect region for this bucket.")
+		return backup_list
-			raise ValueError(e.reason)
-
-		return [(key.name[len(path):], key.size) for key in bucket.list(prefix=path)]
 	elif target.scheme == 'b2':
 		from b2sdk.v1 import InMemoryAccountInfo, B2Api
 		from b2sdk.v1.exception import NonExistentBucket

management/daemon.py

@@ -121,8 +121,10 @@ def index():
 	no_users_exist = (len(get_mail_users(env)) == 0)
 	no_admins_exist = (len(get_admins(env)) == 0)
 
-	import boto.s3
+	import boto3.s3
-	backup_s3_hosts = [(r.name, r.endpoint) for r in boto.s3.regions()]
+	from urllib.parse import urlparse
+	backup_s3_hosts = [(r, f"s3.{r}.amazonaws.com") for r in boto3.session.Session().get_available_regions('s3')]
+
 
 	return render_template('index.html',
 		hostname=env['PRIMARY_HOSTNAME'],

management/status_checks.py

@@ -715,7 +715,7 @@ def check_mail_domain(domain, env, output):
 		output.print_ok(good_news)
 
 		# Check MTA-STS policy.
-		loop = asyncio.get_event_loop()
+		loop = asyncio.new_event_loop()
 		sts_resolver = postfix_mta_sts_resolver.resolver.STSResolver(loop=loop)
 		valid, policy = loop.run_until_complete(sts_resolver.resolve(domain))
 		if valid == postfix_mta_sts_resolver.resolver.STSFetchResult.VALID: