Work on retry logic, more logging

k8s_gitlab_borg/__main__.py

@@ -18,6 +18,7 @@ from base64 import b64decode
 from minio import Minio
 from http.client import IncompleteRead
 from time import sleep
+from random import uniform
 
 from datetime import datetime, timezone, timedelta
 from dateutil.tz import gettz, UTC
@@ -408,25 +409,36 @@ def main():
                     offset = 0
                     retries = 10
                     while True:
+                        logger.info(
+                            "Downloading ‘%s’ (%s, ‘%s’)...",
+                            backup.filename,
+                            backup.creation,
+                            backup.version_id,
+                        )
                         download = minio.get_object(
                             bucket_name="gitlab-backups",
                             object_name=backup.filename,
                             version_id=backup.version_id,
                             offset=offset,
                         )
-                        download_stream = download.stream(decode_content=True)
+                        download_stream = download.stream(amt=1024 * 1024)
                         try:
                             while chunk := next(download_stream, b""):
+                                logger.debug("Read chunk of %d bytes", len(chunk))
                                 offset += len(chunk)
                                 retries = 10
                                 yield chunk
                             else:
                                 break
                         except IncompleteRead as e:
+                            logger.warn(
+                                "IncompleteRead, retries=%d", retries, exc_info=True
+                            )
                             if retries <= 0:
+                                logger.error("Max retries exceeded")
                                 raise e
                             retries -= 1
-                            sleep(10)
+                            sleep(uniform(0, 10))
 
                 download_stream = download_iter()