uniworx/k8s-gitlab-borg
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

extend, not append

Browse Source
This commit is contained in:
Gregor Kleen 2023-05-29 17:10:40 +02:00
parent 6be4f57a49
commit 202e2e391f
1 changed files with 125 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
k8s_gitlab_borg/__main__.py
View File

@ -404,9 +404,8 @@ def main():
import_args += [archive_name, "-"] import_args += [archive_name, "-"]
logger.debug("%s", {"import_args": import_args, "env": env}) logger.debug("%s", {"import_args": import_args, "env": env})
try:
def download_iter(): def download_iter():
try:
offset = 0 offset = 0
retries = 10 retries = 10
while True: while True:
@ -416,6 +415,9 @@ def main():
backup.creation, backup.creation,
backup.version_id, backup.version_id,
) )
if download:
download.close()
download.release_conn()
download = minio.get_object( download = minio.get_object(
bucket_name="gitlab-backups", bucket_name="gitlab-backups",
object_name=backup.filename, object_name=backup.filename,
@ -443,6 +445,9 @@ def main():
raise e raise e
retries -= 1 retries -= 1
sleep(uniform(0, 10)) sleep(uniform(0, 10))
finally:
download.close()
download.release_conn()
download_stream = download_iter() download_stream = download_iter()
@ -469,17 +474,11 @@ def main():
) )
poll.register( poll.register(
proc.stdout, proc.stdout,
select.POLLIN select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR,
| select.POLLPRI
| select.POLLHUP
| select.POLLERR,
) )
poll.register( poll.register(
proc.stderr, proc.stderr,
select.POLLIN select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR,
| select.POLLPRI
| select.POLLHUP
| select.POLLERR,
) )
pollc = 3 pollc = 3
logger.debug("First poll...") logger.debug("First poll...")
@ -512,7 +511,7 @@ def main():
logger.debug("Reading from stdout...") logger.debug("Reading from stdout...")
if chunk := proc.stdout.read(PIPE_BUF): if chunk := proc.stdout.read(PIPE_BUF):
logger.debug("Done, length %d", len(chunk)) logger.debug("Done, length %d", len(chunk))
stdout_line_buffer.append(chunk) stdout_line_buffer.extend(chunk)
while True: while True:
line, sep, rest = stdout_line_buffer.partition( line, sep, rest = stdout_line_buffer.partition(
b"\n" b"\n"
@ -526,7 +525,7 @@ def main():
logger.debug("Reading from stderr...") logger.debug("Reading from stderr...")
if chunk := proc.stderr.read(PIPE_BUF): if chunk := proc.stderr.read(PIPE_BUF):
logger.debug("Done, length %d", len(chunk)) logger.debug("Done, length %d", len(chunk))
stderr_line_buffer.append(chunk) stderr_line_buffer.extend(chunk)
while True: while True:
line, sep, rest = stderr_line_buffer.partition( line, sep, rest = stderr_line_buffer.partition(
b"\n" b"\n"
@ -575,9 +574,6 @@ def main():
logger.debug("Done") logger.debug("Done")
if ret != 0: if ret != 0:
raise Exception(f"borg subprocess exited with returncode {ret}") raise Exception(f"borg subprocess exited with returncode {ret}")
finally:
download.close()
download.release_conn()
copied.add(backup) copied.add(backup)