uniworx/k8s-gitlab-borg
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

retry download if minio closes connection

Browse Source
This commit is contained in:
Gregor Kleen 2023-05-29 14:40:35 +02:00
parent ae53231065
commit ace6dbd49f
1 changed files with 28 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
k8s_gitlab_borg/__main__.py
View File

@ -16,6 +16,8 @@ import logging
from base64 import b64decode from base64 import b64decode
from minio import Minio from minio import Minio
from http.client import IncompleteRead
from time import sleep
from datetime import datetime, timezone, timedelta from datetime import datetime, timezone, timedelta
from dateutil.tz import gettz, UTC from dateutil.tz import gettz, UTC
@ -401,12 +403,32 @@ def main():
logger.debug("%s", {"import_args": import_args, "env": env}) logger.debug("%s", {"import_args": import_args, "env": env})
try: try:
download = minio.get_object(
bucket_name="gitlab-backups", def download_iter():
object_name=backup.filename, offset = 0
version_id=backup.version_id, retries = 10
) while True:
download_stream = download.stream(decode_content=True) download = minio.get_object(
bucket_name="gitlab-backups",
object_name=backup.filename,
version_id=backup.version_id,
offset=offset,
)
download_stream = download.stream(decode_content=True)
try:
while chunk := next(download_stream, b""):
offset += len(chunk)
retries = 10
yield chunk
else:
break
except IncompleteRead as e:
if retries <= 0:
raise e
retries -= 1
sleep(10)
download_stream = download_iter()
with subprocess.Popen( with subprocess.Popen(
import_args, import_args,