uniworx/k8s-gitlab-borg
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

reduce logging

Browse Source
This commit is contained in:
Gregor Kleen 2023-05-29 17:34:36 +02:00
parent 8044d32ad8
commit aa1f9dbd1c
1 changed files with 29 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
k8s_gitlab_borg/__main__.py
View File

@ -428,7 +428,7 @@ def main():
download_stream = download.stream(amt=1024 * 1024) download_stream = download.stream(amt=1024 * 1024)
try: try:
while chunk := next(download_stream, b""): while chunk := next(download_stream, b""):
logger.debug("Read chunk of length %d", len(chunk)) # logger.debug("Read chunk of length %d", len(chunk))
offset += len(chunk) offset += len(chunk)
retries = 10 retries = 10
yield chunk yield chunk
@ -482,43 +482,44 @@ def main():
select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR, select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR,
) )
pollc = 3 pollc = 3
logger.debug("First poll...") # logger.debug("First poll...")
events = poll.poll() events = poll.poll()
logger.debug("Done, %d event(s)", len(events)) # logger.debug("Done, %d event(s)", len(events))
pipe_buffer = b"" pipe_buffer = b""
stdout_line_buffer = bytearray(b"") stdout_line_buffer = bytearray(b"")
stderr_line_buffer = bytearray(b"") stderr_line_buffer = bytearray(b"")
newline_trans = bytearray.maketrans(b"\r", b"\n") newline_trans = bytearray.maketrans(b"\r", b"\n")
while pollc > 0 and len(events) > 0: while pollc > 0 and len(events) > 0:
for rfd, event in events: for rfd, event in events:
logger.debug("rfd=%d, event=%x", rfd, event) # logger.debug("rfd=%d, event=%x", rfd, event)
if event & select.POLLOUT: if event & select.POLLOUT:
if rfd == proc.stdin.fileno(): if rfd == proc.stdin.fileno():
if chunk := pipe_buffer[:PIPE_BUF]: if chunk := pipe_buffer[:PIPE_BUF]:
logger.debug( # logger.debug(
"Writing chunk of length %d...", len(chunk) # "Writing chunk of length %d...", len(chunk)
) # )
proc.stdin.buffer.write(chunk) proc.stdin.buffer.write(chunk)
logger.debug("Done") # logger.debug("Done")
pipe_buffer = pipe_buffer[PIPE_BUF:] pipe_buffer = pipe_buffer[PIPE_BUF:]
elif pipe_buffer := next(download_stream, b""): elif pipe_buffer := next(download_stream, b""):
logger.debug( # logger.debug(
"Accepted chunk of length %d", # "Accepted chunk of length %d",
len(pipe_buffer), # len(pipe_buffer),
) # )
pass
else: else:
proc.stdin.close() proc.stdin.close()
if event & select.POLLIN or event & select.POLLPRI: if event & select.POLLIN or event & select.POLLPRI:
if rfd == proc.stdout.fileno(): if rfd == proc.stdout.fileno():
logger.debug("Reading from stdout...") # logger.debug("Reading from stdout...")
if chunk := proc.stdout.buffer.read(PIPE_BUF): if chunk := proc.stdout.buffer.read(PIPE_BUF):
logger.debug("Done, length %d", len(chunk)) # logger.debug("Done, length %d", len(chunk))
stdout_line_buffer.extend( stdout_line_buffer.extend(
chunk.translate(newline_trans) chunk.translate(newline_trans)
) )
logger.debug( # logger.debug(
"Buffer at length %d", len(stdout_line_buffer) # "Buffer at length %d", len(stdout_line_buffer)
) # )
while True: while True:
line, sep, rest = stdout_line_buffer.partition( line, sep, rest = stdout_line_buffer.partition(
b"\n" b"\n"
@ -532,15 +533,15 @@ def main():
else: else:
break break
if rfd == proc.stderr.fileno(): if rfd == proc.stderr.fileno():
logger.debug("Reading from stderr...") # logger.debug("Reading from stderr...")
if chunk := proc.stderr.buffer.read(PIPE_BUF): if chunk := proc.stderr.buffer.read(PIPE_BUF):
logger.debug("Done, length %d", len(chunk)) # logger.debug("Done, length %d", len(chunk))
stderr_line_buffer.extend( stderr_line_buffer.extend(
chunk.translate(newline_trans) chunk.translate(newline_trans)
) )
logger.debug( # logger.debug(
"Buffer at length %d", len(stderr_line_buffer) # "Buffer at length %d", len(stderr_line_buffer)
) # )
while True: while True:
line, sep, rest = stderr_line_buffer.partition( line, sep, rest = stderr_line_buffer.partition(
b"\n" b"\n"
@ -571,11 +572,12 @@ def main():
pollc -= 1 pollc -= 1
if pollc > 0: if pollc > 0:
logger.debug("Poll...") # logger.debug("Poll...")
events = poll.poll() events = poll.poll()
logger.debug("Done, %d event(s)", len(events)) # logger.debug("Done, %d event(s)", len(events))
else: else:
logger.debug("Nothing left to poll") # logger.debug("Nothing left to poll")
pass
if stdout_line_buffer: if stdout_line_buffer:
for line in stdout_line_buffer.split(b"\n"): for line in stdout_line_buffer.split(b"\n"):
@ -587,9 +589,9 @@ def main():
for handler in proc_logger.handlers: for handler in proc_logger.handlers:
handler.flush() handler.flush()
logger.debug("Waiting on subprocess...") # logger.debug("Waiting on subprocess...")
ret = proc.wait() ret = proc.wait()
logger.debug("Done") # logger.debug("Done")
if ret != 0: if ret != 0:
raise Exception(f"borg subprocess exited with returncode {ret}") raise Exception(f"borg subprocess exited with returncode {ret}")