Auto merge of #128459 - lolbinarycat:wget, r=Kobzol

make bootstrap use wget if available

this improves reliability when downloading over unreliable connections.

without this change, it was basically impossible for me to run the bootstrap tests.

Author: bors

src/bootstrap/bootstrap.py

@@ -44,7 +44,6 @@ def eprint(*args, **kwargs):
     kwargs["file"] = sys.stderr
     print(*args, **kwargs)
 
-
 def get(base, url, path, checksums, verbose=False):
     with tempfile.NamedTemporaryFile(delete=False) as temp_file:
         temp_path = temp_file.name
@@ -89,6 +88,8 @@ def download(path, url, probably_big, verbose):
             eprint("\nspurious failure, trying again")
     _download(path, url, probably_big, verbose, False)
 
+def has_wget():
+    return require(["wget", "--version"], exit=False) is not None
 
 def _download(path, url, probably_big, verbose, exception):
     # Try to use curl (potentially available on win32
@@ -100,22 +101,45 @@ def _download(path, url, probably_big, verbose, exception):
         eprint("downloading {}".format(url))
 
     try:
-        if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ:
-            option = "-#"
+        if has_wget():
+            # options should be kept in sync with
+            # src/bootstrap/src/core/download.rs
+            # for consistancy
+            # these flags should also be as close as possible to the behavior
+            # of curl (except for wget's superior handling of surious network
+            # errors)
+            # curl's -R and -f are wget's default behavior.
+            run(["wget",
+                 "--connect-timeout=30",
+                 "--read-timeout=30",
+                 "--tries=3",
+                 "--show-progress",
+                 "-O", path, url],
+                verbose=verbose,
+                exception=True,
+            )
         else:
-            option = "-s"
-        # If curl is not present on Win32, we should not sys.exit
-        #   but raise `CalledProcessError` or `OSError` instead
-        require(["curl", "--version"], exception=platform_is_win32())
-        run(["curl", option,
-            "-L", # Follow redirect.
-            "-y", "30", "-Y", "10",    # timeout if speed is < 10 bytes/sec for > 30 seconds
-            "--connect-timeout", "30",  # timeout if cannot connect within 30 seconds
-            "-o", path,
-            "--retry", "3", "-SRf", url],
-            verbose=verbose,
-            exception=True, # Will raise RuntimeError on failure
-        )
+            if (probably_big or verbose) and "GITHUB_ACTIONS" not in os.environ:
+                option = "-#"
+            else:
+                option = "-s"
+                # If curl is not present on Win32, we should not sys.exit
+                #   but raise `CalledProcessError` or `OSError` instead
+                require(["curl", "--version"], exception=platform_is_win32())
+                run(["curl", option,
+                     "-L", # Follow redirect.
+                     # timeout if speed is < 10 bytes/sec for > 30 seconds
+                     "-y", "30", "-Y", "10",
+                     # timeout if cannot connect within 30 seconds
+                     "--connect-timeout", "30",
+                     "-o", path,
+                     # -S: show errors, even if -s is specified
+                     # -R: set timestamp of downloaded file to that of the server
+                     # -f: fail on http error
+                     "--retry", "3", "-SRf", url],
+                    verbose=verbose,
+                    exception=True, # Will raise RuntimeError on failure
+                    )
     except (subprocess.CalledProcessError, OSError, RuntimeError):
         # see http://serverfault.com/questions/301128/how-to-download
         if platform_is_win32():
@@ -129,6 +153,7 @@ def _download(path, url, probably_big, verbose, exception):
             raise
 
 
+
 def verify(path, expected, verbose):
     """Check if the sha256 sum of the given path is valid"""
     if verbose:

src/bootstrap/src/core/download.rs

@@ -210,6 +210,8 @@ impl Config {
         println!("downloading {url}");
         // Try curl. If that fails and we are on windows, fallback to PowerShell.
         let mut curl = command("curl");
+        // build the arguments for curl and wget simultaneously
+        let mut wget = command("wget");
         curl.args([
             "-y",
             "30",
@@ -223,14 +225,27 @@ impl Config {
             "3",
             "-SRf",
         ]);
+        // options should be kept in sync with
+        // src/bootstrap/bootstrap.py
+        // for consistancy
+        wget.args([
+            "--connect-timeout=30",
+            "--read-timeout=30",
+            "--tries=3",
+            "-O",
+            tempfile.to_str().unwrap(),
+        ]);
         // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful.
         if CiEnv::is_ci() {
             curl.arg("-s");
         } else {
             curl.arg("--progress-bar");
+            wget.arg("--show-progress");
         }
         curl.arg(url);
-        if !self.check_run(&mut curl) {
+        wget.arg(url);
+        curl.mark_as_executed();
+        if !(self.check_run(&mut wget) || self.check_run(&mut curl)) {
             if self.build.contains("windows-msvc") {
                 eprintln!("Fallback to PowerShell");
                 for _ in 0..3 {
@@ -346,7 +361,9 @@ impl Config {
             println!(
                 "invalid checksum: \n\
                 found:    {checksum}\n\
-                expected: {expected}",
+                expected: {expected}\n\
+                path: {}",
+                path.display(),
             );
         }