uniworx/k8s-gitlab-borg
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

reduce logging

Browse Source
This commit is contained in:
Gregor Kleen 2023-05-29 17:34:36 +02:00
parent 8044d32ad8
commit aa1f9dbd1c
1 changed files with 29 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
k8s_gitlab_borg/__main__.py
View File

@ -428,7 +428,7 @@ def main():
download_stream = download.stream(amt=1024 * 1024)
try:
while chunk := next(download_stream, b""):
logger.debug("Read chunk of length %d", len(chunk))
# logger.debug("Read chunk of length %d", len(chunk))
offset += len(chunk)
retries = 10
yield chunk
@ -482,43 +482,44 @@ def main():
select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR,
)
pollc = 3
logger.debug("First poll...")
# logger.debug("First poll...")
events = poll.poll()
logger.debug("Done, %d event(s)", len(events))
# logger.debug("Done, %d event(s)", len(events))
pipe_buffer = b""
stdout_line_buffer = bytearray(b"")
stderr_line_buffer = bytearray(b"")
newline_trans = bytearray.maketrans(b"\r", b"\n")
while pollc > 0 and len(events) > 0:
for rfd, event in events:
logger.debug("rfd=%d, event=%x", rfd, event)
# logger.debug("rfd=%d, event=%x", rfd, event)
if event & select.POLLOUT:
if rfd == proc.stdin.fileno():
if chunk := pipe_buffer[:PIPE_BUF]:
logger.debug(
"Writing chunk of length %d...", len(chunk)
)
# logger.debug(
# "Writing chunk of length %d...", len(chunk)
# )
proc.stdin.buffer.write(chunk)
logger.debug("Done")
# logger.debug("Done")
pipe_buffer = pipe_buffer[PIPE_BUF:]
elif pipe_buffer := next(download_stream, b""):
logger.debug(
"Accepted chunk of length %d",
len(pipe_buffer),
)
# logger.debug(
# "Accepted chunk of length %d",
# len(pipe_buffer),
# )
pass
else:
proc.stdin.close()
if event & select.POLLIN or event & select.POLLPRI:
if rfd == proc.stdout.fileno():
logger.debug("Reading from stdout...")
# logger.debug("Reading from stdout...")
if chunk := proc.stdout.buffer.read(PIPE_BUF):
logger.debug("Done, length %d", len(chunk))
# logger.debug("Done, length %d", len(chunk))
stdout_line_buffer.extend(
chunk.translate(newline_trans)
)
logger.debug(
"Buffer at length %d", len(stdout_line_buffer)
)
# logger.debug(
# "Buffer at length %d", len(stdout_line_buffer)
# )
while True:
line, sep, rest = stdout_line_buffer.partition(
b"\n"
@ -532,15 +533,15 @@ def main():
else:
break
if rfd == proc.stderr.fileno():
logger.debug("Reading from stderr...")
# logger.debug("Reading from stderr...")
if chunk := proc.stderr.buffer.read(PIPE_BUF):
logger.debug("Done, length %d", len(chunk))
# logger.debug("Done, length %d", len(chunk))
stderr_line_buffer.extend(
chunk.translate(newline_trans)
)
logger.debug(
"Buffer at length %d", len(stderr_line_buffer)
)
# logger.debug(
# "Buffer at length %d", len(stderr_line_buffer)
# )
while True:
line, sep, rest = stderr_line_buffer.partition(
b"\n"
@ -571,11 +572,12 @@ def main():
pollc -= 1
if pollc > 0:
logger.debug("Poll...")
# logger.debug("Poll...")
events = poll.poll()
logger.debug("Done, %d event(s)", len(events))
# logger.debug("Done, %d event(s)", len(events))
else:
logger.debug("Nothing left to poll")
# logger.debug("Nothing left to poll")
pass
if stdout_line_buffer:
for line in stdout_line_buffer.split(b"\n"):
@ -587,9 +589,9 @@ def main():
for handler in proc_logger.handlers:
handler.flush()
logger.debug("Waiting on subprocess...")
# logger.debug("Waiting on subprocess...")
ret = proc.wait()
logger.debug("Done")
# logger.debug("Done")
if ret != 0:
raise Exception(f"borg subprocess exited with returncode {ret}")