uniworx/k8s-gitlab-borg
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

more logging, PAGESIZE chunks to STDIN

Browse Source
This commit is contained in:
Gregor Kleen 2023-05-29 15:50:03 +02:00
parent 86767b2e39
commit 47eb6ee63b
1 changed files with 17 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
k8s_gitlab_borg/__main__.py
View File

@ -65,6 +65,8 @@ TIME_PATTERNS = OrderedDict(
BACKUP_PATTERN = re.compile(r"^(?P<ts>[0-9]+)_.*_gitlab_backup\.tar$") BACKUP_PATTERN = re.compile(r"^(?P<ts>[0-9]+)_.*_gitlab_backup\.tar$")
BUFFER_SIZE = os.sysconf("SC_PAGESIZE")
@dataclass(eq=True, order=True, frozen=True) @dataclass(eq=True, order=True, frozen=True)
class GitlabBackup: class GitlabBackup:
@ -424,10 +426,12 @@ def main():
download_stream = download.stream(amt=1024 * 1024) download_stream = download.stream(amt=1024 * 1024)
try: try:
while chunk := next(download_stream, b""): while chunk := next(download_stream, b""):
# logger.debug("Read chunk of %d bytes", len(chunk)) logger.debug("Read chunk of %d bytes", len(chunk))
offset += len(chunk) offset += len(chunk)
retries = 10 retries = 10
yield chunk while pagesize_chunk := chunk[:BUFFER_SIZE]:
yield pagesize_chunk
chunk = chunk[BUFFER_SIZE:]
else: else:
break break
except IncompleteRead as e: except IncompleteRead as e:
@ -462,14 +466,18 @@ def main():
poll.register(proc.stdin, select.POLLOUT | select.POLLHUP) poll.register(proc.stdin, select.POLLOUT | select.POLLHUP)
poll.register(proc.stdout, select.POLLIN | select.POLLHUP) poll.register(proc.stdout, select.POLLIN | select.POLLHUP)
poll.register(proc.stderr, select.POLLIN | select.POLLHUP) poll.register(proc.stderr, select.POLLIN | select.POLLHUP)
pollc = 2 pollc = 3
events = poll.poll() events = poll.poll()
while pollc > 0 and len(events) > 0: while pollc > 0 and len(events) > 0:
for rfd, event in events: for rfd, event in events:
if event & select.POLLOUT: if event & select.POLLOUT:
if rfd == proc.stdin.fileno(): if rfd == proc.stdin.fileno():
if chunk := next(download_stream, b""): if chunk := next(download_stream, b""):
logger.debug(
"Writing chunk of length %d...", len(chunk)
)
proc.stdin.buffer.write(chunk) proc.stdin.buffer.write(chunk)
logger.debug("Done")
else: else:
proc.stdin.close() proc.stdin.close()
if event & select.POLLIN: if event & select.POLLIN:
@ -480,6 +488,12 @@ def main():
if line := proc.stderr.readline(): if line := proc.stderr.readline():
stderr_logger.info(line[:-1]) stderr_logger.info(line[:-1])
if event & select.POLLHUP: if event & select.POLLHUP:
if rfd == proc.stdin.fileno():
logger.debug("STDIN closed")
if rfd == proc.stdout.fileno():
logger.debug("STDOUT closed")
if rfd == proc.stderr.fileno():
logger.debug("STDERR closed")
poll.unregister(rfd) poll.unregister(rfd)
pollc -= 1 pollc -= 1