From b18065749dbe9970b5ff23094363dce590bf3fec Mon Sep 17 00:00:00 2001 From: Dominik Schubert Date: Thu, 2 Feb 2023 10:50:47 +0100 Subject: [PATCH 1/3] add download retry and log download errors in installer --- localstack/packages/core.py | 1 + localstack/utils/archives.py | 23 ++++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/localstack/packages/core.py b/localstack/packages/core.py index 4dca8cc7fbddb..dd599425288af 100644 --- a/localstack/packages/core.py +++ b/localstack/packages/core.py @@ -106,6 +106,7 @@ def _install(self, target: InstallTarget) -> None: archive_name = os.path.basename(download_url) download_and_extract( download_url, + retries=3, tmp_archive=os.path.join(config.dirs.tmp, archive_name), target_dir=target_directory, ) diff --git a/localstack/utils/archives.py b/localstack/utils/archives.py index 361dbdfc39a55..171256133e55a 100644 --- a/localstack/utils/archives.py +++ b/localstack/utils/archives.py @@ -185,12 +185,29 @@ def download_and_extract(archive_url, target_dir, retries=0, sleep=3, tmp_archiv if not os.path.exists(tmp_archive) or os.path.getsize(tmp_archive) <= 0: # create temporary placeholder file, to avoid duplicate parallel downloads save_file(tmp_archive, "") - for i in range(retries + 1): + + current_try = 1 + while current_try <= retries + 1: try: download(archive_url, tmp_archive) break - except Exception: - time.sleep(sleep) + except Exception as e: + LOG.warning( + "Attempt %d. Failed to download archive from %s: %s", + current_try, + archive_url, + e, + ) + # only sleep between retries, not after the last one + if current_try < retries: + time.sleep(sleep) + finally: + current_try += 1 + + # if the temporary file we created above hasn't been replaced, we assume failure + if os.path.getsize(tmp_archive) <= 0: + raise Exception("Failed to download archive from %s: . Retries exhausted", archive_url) + if ext == ".zip": unzip(tmp_archive, target_dir) elif ext in ( From a850bc344080d4af6e420b127adf8faa34fcfbf3 Mon Sep 17 00:00:00 2001 From: Dominik Schubert Date: Thu, 2 Feb 2023 11:03:47 +0100 Subject: [PATCH 2/3] fix off-by-1 --- localstack/utils/archives.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/localstack/utils/archives.py b/localstack/utils/archives.py index 171256133e55a..185fae2b8c30c 100644 --- a/localstack/utils/archives.py +++ b/localstack/utils/archives.py @@ -199,7 +199,7 @@ def download_and_extract(archive_url, target_dir, retries=0, sleep=3, tmp_archiv e, ) # only sleep between retries, not after the last one - if current_try < retries: + if current_try <= retries: time.sleep(sleep) finally: current_try += 1 From 1f109747bd9e27950be874c9e69be94d910cac26 Mon Sep 17 00:00:00 2001 From: Dominik Schubert Date: Fri, 3 Feb 2023 11:44:00 +0100 Subject: [PATCH 3/3] exchange while for for loop --- localstack/utils/archives.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/localstack/utils/archives.py b/localstack/utils/archives.py index 185fae2b8c30c..dfba8d3c9aafc 100644 --- a/localstack/utils/archives.py +++ b/localstack/utils/archives.py @@ -186,23 +186,20 @@ def download_and_extract(archive_url, target_dir, retries=0, sleep=3, tmp_archiv # create temporary placeholder file, to avoid duplicate parallel downloads save_file(tmp_archive, "") - current_try = 1 - while current_try <= retries + 1: + for i in range(retries + 1): try: download(archive_url, tmp_archive) break except Exception as e: LOG.warning( "Attempt %d. Failed to download archive from %s: %s", - current_try, + i + 1, archive_url, e, ) # only sleep between retries, not after the last one - if current_try <= retries: + if i < retries: time.sleep(sleep) - finally: - current_try += 1 # if the temporary file we created above hasn't been replaced, we assume failure if os.path.getsize(tmp_archive) <= 0: