diff --git a/k8s_gitlab_borg/__main__.py b/k8s_gitlab_borg/__main__.py index 1473de5..2604369 100644 --- a/k8s_gitlab_borg/__main__.py +++ b/k8s_gitlab_borg/__main__.py @@ -65,6 +65,8 @@ TIME_PATTERNS = OrderedDict( BACKUP_PATTERN = re.compile(r"^(?P[0-9]+)_.*_gitlab_backup\.tar$") +BUFFER_SIZE = os.sysconf("SC_PAGESIZE") + @dataclass(eq=True, order=True, frozen=True) class GitlabBackup: @@ -424,10 +426,12 @@ def main(): download_stream = download.stream(amt=1024 * 1024) try: while chunk := next(download_stream, b""): - # logger.debug("Read chunk of %d bytes", len(chunk)) + logger.debug("Read chunk of %d bytes", len(chunk)) offset += len(chunk) retries = 10 - yield chunk + while pagesize_chunk := chunk[:BUFFER_SIZE]: + yield pagesize_chunk + chunk = chunk[BUFFER_SIZE:] else: break except IncompleteRead as e: @@ -462,14 +466,18 @@ def main(): poll.register(proc.stdin, select.POLLOUT | select.POLLHUP) poll.register(proc.stdout, select.POLLIN | select.POLLHUP) poll.register(proc.stderr, select.POLLIN | select.POLLHUP) - pollc = 2 + pollc = 3 events = poll.poll() while pollc > 0 and len(events) > 0: for rfd, event in events: if event & select.POLLOUT: if rfd == proc.stdin.fileno(): if chunk := next(download_stream, b""): + logger.debug( + "Writing chunk of length %d...", len(chunk) + ) proc.stdin.buffer.write(chunk) + logger.debug("Done") else: proc.stdin.close() if event & select.POLLIN: @@ -480,6 +488,12 @@ def main(): if line := proc.stderr.readline(): stderr_logger.info(line[:-1]) if event & select.POLLHUP: + if rfd == proc.stdin.fileno(): + logger.debug("STDIN closed") + if rfd == proc.stdout.fileno(): + logger.debug("STDOUT closed") + if rfd == proc.stderr.fileno(): + logger.debug("STDERR closed") poll.unregister(rfd) pollc -= 1