#71 - Robustness: Have fallbacks if http://cgit.drupalcode.org is not available (#100)

Author: vijaycs85

.scenarios.lock/phpunit4/composer.json

@@ -4,7 +4,7 @@
     "type": "composer-plugin",
     "license": "GPL-2.0-or-later",
     "require": {
-        "php": ">=5.4.5",
+        "php": "^5.5.9|>=7.0.8",
         "composer-plugin-api": "^1.0.0",
         "composer/semver": "^1.4"
     },

.scenarios.lock/phpunit4/composer.lock

@@ -37,6 +37,7 @@ install:
 before_script:
   - git config --global user.email "[email protected]"
   - git config --global user.name "Travis CI Test"
+  - export COMPOSER_PROCESS_TIMEOUT=600
 
 script:
   - composer test

.travis.yml

@@ -51,6 +51,21 @@ The `source` option may be used to specify the URL to download the
 scaffold files from; the default source is drupal.org. The literal string
 `{version}` in the `source` option is replaced with the current version of
 Drupal core being updated prior to download.
+You can also define `source` as an array to have fallbacks in case of
+any HTTP issues.
+
+```json
+{
+  "extra": {
+    "drupal-scaffold": {
+      "source": [
+        "https://cgit.drupalcode.org/drupal/plain/{path}?h={version}",
+        "https://raw.githubusercontent.com/drupal/drupal/{version}/{path}"
+      ]
+    }
+  }
+}
+```
 
 With the `drupal-scaffold` option `excludes`, you can provide additional paths
 that should not be copied or overwritten. The plugin provides no excludes by

README.md

@@ -49,13 +49,19 @@ class FileFetcher {
    */
   protected $fs;
 
+  /**
+   * @var array
+   *
+   * A list of potential errors.
+   */
+  protected $errors = [];
+
   /**
    * Constructs this FileFetcher object.
    */
-  public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInterface $io, $progress = TRUE) {
+  public function __construct(RemoteFilesystem $remoteFilesystem, IOInterface $io, $progress = TRUE) {
     $this->remoteFilesystem = $remoteFilesystem;
     $this->io = $io;
-    $this->source = $source;
     $this->fs = new Filesystem();
     $this->progress = $progress;
   }
@@ -64,23 +70,39 @@ public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInter
    * Downloads all required files and writes it to the file system.
    */
   public function fetch($version, $destination, $override) {
+    $errors = [];
+
     foreach ($this->filenames as $sourceFilename => $filename) {
       $target = "$destination/$filename";
       if ($override || !file_exists($target)) {
         $url = $this->getUri($sourceFilename, $version);
         $this->fs->ensureDirectoryExists($destination . '/' . dirname($filename));
+
         if ($this->progress) {
           $this->io->writeError("  - <info>$filename</info> (<comment>$url</comment>): ", FALSE);
-          $this->remoteFilesystem->copy($url, $url, $target, $this->progress);
-          // Used to put a new line because the remote file system does not put
-          // one.
+          try {
+            $this->remoteFilesystem->copy($url, $url, $target, $this->progress);
+          } catch(\Exception $e) {
+            $errors[] = $url;
+          }
+          // New line because the remoteFilesystem does not put one.
           $this->io->writeError('');
         }
         else {
-          $this->remoteFilesystem->copy($url, $url, $target, $this->progress);
+          try {
+            $this->remoteFilesystem->copy($url, $url, $target, $this->progress);
+          } catch(\Exception $e) {
+            $errors[] = $url;
+          }
         }
       }
     }
+
+    if ($errors) {
+      $this->addError('Failed to download: ' . "\r\n" . implode("\r\n", $errors));
+      return FALSE;
+    }
+    return TRUE;
   }
 
   /**
@@ -90,6 +112,27 @@ public function setFilenames(array $filenames) {
     $this->filenames = $filenames;
   }
 
+  /**
+   * Set source.
+   */
+  public function setSource($source) {
+    $this->source = $source;
+  }
+
+  /**
+   * Set error.
+   */
+  public function addError($error) {
+    $this->errors[] = $error;
+  }
+
+  /**
+   * Get errors.
+   */
+  public function getErrors() {
+    return $this->errors;
+  }
+
   /**
    * Replace filename and version in the source pattern with their values.
    */

composer.lock

@@ -149,6 +149,7 @@ public function downloadScaffold() {
     // Collect options, excludes and settings files.
     $options = $this->getOptions();
     $files = array_diff($this->getIncludes(), $this->getExcludes());
+    $files = array_combine($files, $files);
 
     // Call any pre-scaffold scripts that may be defined.
     $dispatcher = new EventDispatcher($this->composer, $this->io);
@@ -158,12 +159,39 @@ public function downloadScaffold() {
 
     $remoteFs = new RemoteFilesystem($this->io);
 
-    $fetcher = new PrestissimoFileFetcher($remoteFs, $options['source'], $this->io, $this->progress, $this->composer->getConfig());
-    $fetcher->setFilenames(array_combine($files, $files));
-    $fetcher->fetch($version, $webroot, TRUE);
+    $fetcher = new PrestissimoFileFetcher($remoteFs, $this->io, $this->progress, $this->composer->getConfig());
+    $sources = (array) $options['source'];
+    $all_succeeded = FALSE;
 
-    $fetcher->setFilenames($this->getInitial());
-    $fetcher->fetch($version, $webroot, FALSE);
+    do {
+      $source = current($sources);
+
+      $fetcher->setSource($source);
+
+      $fetcher->setFilenames($files);
+      if ($fetcher->fetch($version, $webroot, TRUE)) {
+        $fetcher->setFilenames($this->getInitial());
+        if ($fetcher->fetch($version, $webroot, FALSE)) {
+          $all_succeeded = TRUE;
+          break;
+        }
+      }
+
+      // If here, it means that the fetch for this source has failed.
+      $next_source = next($sources);
+
+      $this->io->writeError('');
+      $this->io->writeError("  - Has failed with the " . (!$next_source ? 'last ' : '') . "source: <error>$source</error>", TRUE);
+      if ($next_source) {
+        $this->io->writeError("  - Now trying with the source: <warning>$next_source</warning>", TRUE);
+      }
+      $this->io->writeError('');
+
+    } while($next_source);
+
+    if (!$all_succeeded) {
+      throw new \Exception(implode("\r\n\r\n", $fetcher->getErrors()));
+    }
 
     // Call post-scaffold scripts.
     $dispatcher->dispatch(self::POST_DRUPAL_SCAFFOLD_CMD);
@@ -343,8 +371,10 @@ protected function getOptions() {
       'excludes' => [],
       'includes' => [],
       'initial' => [],
-      'source' => 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}',
-      // Github: https://raw.githubusercontent.com/drupal/drupal/{version}/{path}
+      'source' => [
+        'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}',
+        'https://raw.githubusercontent.com/drupal/drupal/{version}/{path}'
+      ],
     ];
     return $options;
   }

src/FileFetcher.php

@@ -22,8 +22,8 @@ class PrestissimoFileFetcher extends FileFetcher {
   /**
    * Constructs this PrestissimoFileFetcher object.
    */
-  public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInterface $io, $progress = TRUE, Config $config) {
-    parent::__construct($remoteFilesystem, $source, $io, $progress);
+  public function __construct(RemoteFilesystem $remoteFilesystem, IOInterface $io, $progress = TRUE, Config $config) {
+    parent::__construct($remoteFilesystem, $io, $progress);
     $this->config = $config;
   }
 
@@ -32,10 +32,9 @@ public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInter
    */
   public function fetch($version, $destination, $override) {
     if (class_exists(CurlMulti::class)) {
-      $this->fetchWithPrestissimo($version, $destination, $override);
-      return;
+      return $this->fetchWithPrestissimo($version, $destination, $override);
     }
-    parent::fetch($version, $destination, $override);
+    return parent::fetch($version, $destination, $override);
   }
 
   /**
@@ -57,7 +56,7 @@ protected function fetchWithPrestissimo($version, $destination, $override) {
     $errors = [];
     $totalCnt = count($requests);
     if ($totalCnt == 0) {
-      return;
+      return TRUE;
     }
 
     $multi = new CurlMulti();
@@ -70,6 +69,9 @@ protected function fetchWithPrestissimo($version, $destination, $override) {
       $failureCnt += $result['failureCnt'];
       if (isset($result['errors'])) {
         $errors += $result['errors'];
+        foreach ($result['errors'] as $url => $error) {
+          $this->io->writeError("  - Downloading <comment>$successCnt</comment>/<comment>$totalCnt</comment>: <info>$url</info> (<error>failed</error>)", TRUE);
+        }
       }
       if ($this->progress) {
         foreach ($result['urls'] as $url) {
@@ -78,10 +80,20 @@ protected function fetchWithPrestissimo($version, $destination, $override) {
       }
     } while ($multi->remain());
 
-    $urls = array_keys($errors);
-    if ($urls) {
-      throw new \Exception('Failed to download ' . implode(", ", $urls));
+    if ($errors) {
+      $this->addError('Failed to download: ' . "\r\n" . implode("\r\n", array_keys($errors)));
+      $errors_extra = [];
+      foreach($errors as $error) {
+        if ($error !== "0: " && !isset($errors_extra[$error])) {
+          $errors_extra[$error] = $error;
+        }
+      }
+      if ($errors_extra) {
+        $this->addError(implode("\r\n", $errors_extra));
+      }
+      return FALSE;
     }
+    return TRUE;
   }
 
 }

src/Handler.php

@@ -57,7 +57,8 @@ protected function ensureDirectoryExistsAndClear($directory) {
   }
 
   public function testFetch() {
-    $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}', new NullIO());
+    $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), new NullIO());
+    $fetcher->setSource('https://cgit.drupalcode.org/drupal/plain/{path}?h={version}');
     $fetcher->setFilenames([
       '.htaccess' => '.htaccess',
       'sites/default/default.settings.php' => 'sites/default/default.settings.php',
@@ -68,7 +69,8 @@ public function testFetch() {
   }
 
   public function testInitialFetch() {
-    $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}', new NullIO());
+    $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), new NullIO());
+    $fetcher->setSource('https://cgit.drupalcode.org/drupal/plain/{path}?h={version}');
     $fetcher->setFilenames([
       'sites/default/default.settings.php' => 'sites/default/settings.php',
     ]);