feat(updater): Cache the new blockmap file and allow customization of the old blockmap file base URL. (electron-userland#9172)

Currently, downloading updates from GitHub's Latest Release does not
support delta updates. This is mainly due to the inability to download
the old map file. Now, by adding a configuration that allows users to
customize the old map file URL, users can define the old map file URL
themselves. This makes it possible to achieve delta updates without
modifying the server side.

Cache the new blockmap file, and during the next update, prioritize
reading it from the local cache. This reduces requests for the blockmap
and allows delta updates even if the old version’s blockmap file is no
longer available on the server.
 
Fix electron-userland#8811

Author: beyondkmp

.changeset/orange-experts-do.md

@@ -0,0 +1,5 @@
+---
+"electron-updater": patch
+---
+
+feat(updater): Cache the new blockmap file and allow customization of the old blockmap file base URL

packages/electron-updater/src/AppUpdater.ts

@@ -15,7 +15,7 @@ import {
 import { randomBytes } from "crypto"
 import { release } from "os"
 import { EventEmitter } from "events"
-import { mkdir, outputFile, readFile, rename, unlink } from "fs-extra"
+import { mkdir, outputFile, readFile, rename, unlink, copyFile, pathExists } from "fs-extra"
 import { OutgoingHttpHeaders } from "http"
 import { load } from "js-yaml"
 import { Lazy } from "lazy-val"
@@ -31,7 +31,7 @@ import { Provider, ProviderPlatform } from "./providers/Provider"
 import type { TypedEmitter } from "tiny-typed-emitter"
 import Session = Electron.Session
 import type { AuthInfo } from "electron"
-import { gunzipSync } from "zlib"
+import { gunzipSync, gzipSync } from "zlib"
 import { blockmapFiles } from "./util"
 import { DifferentialDownloaderOptions } from "./differentialDownloader/DifferentialDownloader"
 import { GenericDifferentialDownloader } from "./differentialDownloader/GenericDifferentialDownloader"
@@ -117,6 +117,18 @@ export abstract class AppUpdater extends (EventEmitter as new () => TypedEmitter
    */
   forceDevUpdateConfig = false
 
+  /**
+   * The base URL of the old block map file.
+   *
+   * When null, the updater will use the base URL of the update file to download the update.
+   * When set, the updater will use this string as the base URL of the old block map file.
+   * Some servers like github cannot download the old block map file from latest release,
+   * so you need to compute the old block map file base URL manually.
+   *
+   * @default null
+   */
+  public previousBlockmapBaseUrlOverride: string | null = null
+
   /**
    * The current application version.
    */
@@ -736,6 +748,10 @@ export abstract class AppUpdater extends (EventEmitter as new () => TypedEmitter
         ...updateInfo,
         downloadedFile: updateFile,
       })
+      const currentBlockMapFile = path.join(cacheDir, "current.blockmap")
+      if (await pathExists(currentBlockMapFile)) {
+        await copyFile(currentBlockMapFile, path.join(downloadedUpdateHelper.cacheDir, "current.blockmap"))
+      }
       return packageFile == null ? [updateFile] : [updateFile, packageFile]
     }
 
@@ -790,7 +806,7 @@ export abstract class AppUpdater extends (EventEmitter as new () => TypedEmitter
       if (this._testOnlyOptions != null && !this._testOnlyOptions.isUseDifferentialDownload) {
         return true
       }
-      const blockmapFileUrls = blockmapFiles(fileInfo.url, this.app.version, downloadUpdateOptions.updateInfoAndProvider.info.version)
+      const blockmapFileUrls = blockmapFiles(fileInfo.url, this.app.version, downloadUpdateOptions.updateInfoAndProvider.info.version, this.previousBlockmapBaseUrlOverride)
       this._logger.info(`Download block maps (old: "${blockmapFileUrls[0]}", new: ${blockmapFileUrls[1]})`)
 
       const downloadBlockMap = async (url: URL): Promise<BlockMap> => {
@@ -824,8 +840,33 @@ export abstract class AppUpdater extends (EventEmitter as new () => TypedEmitter
         downloadOptions.onProgress = it => this.emit(DOWNLOAD_PROGRESS, it)
       }
 
-      const blockMapDataList = await Promise.all(blockmapFileUrls.map(u => downloadBlockMap(u)))
-      await new GenericDifferentialDownloader(fileInfo.info, this.httpExecutor, downloadOptions).download(blockMapDataList[0], blockMapDataList[1])
+      const saveBlockMapToCacheDir = async (blockMapData: BlockMap, cacheDir: string) => {
+        const blockMapFile = path.join(cacheDir, "current.blockmap")
+        await outputFile(blockMapFile, gzipSync(JSON.stringify(blockMapData)))
+      }
+
+      const getBlockMapFromCacheDir = async (cacheDir: string) => {
+        const blockMapFile = path.join(cacheDir, "current.blockmap")
+        try {
+          if (await pathExists(blockMapFile)) {
+            return JSON.parse(gunzipSync(await readFile(blockMapFile)).toString())
+          }
+        } catch (e: any) {
+          this._logger.warn(`Cannot parse blockmap "${blockMapFile}", error: ${e}`)
+        }
+        return null
+      }
+
+      const newBlockMapData = await downloadBlockMap(blockmapFileUrls[1])
+      await saveBlockMapToCacheDir(newBlockMapData, this.downloadedUpdateHelper!.cacheDirForPendingUpdate)
+
+      // get old blockmap from cache dir first, if not found, download it
+      let oldBlockMapData = await getBlockMapFromCacheDir(this.downloadedUpdateHelper!.cacheDir)
+      if (oldBlockMapData == null) {
+        oldBlockMapData = await downloadBlockMap(blockmapFileUrls[0])
+      }
+
+      await new GenericDifferentialDownloader(fileInfo.info, this.httpExecutor, downloadOptions).download(oldBlockMapData, newBlockMapData)
       return false
     } catch (e: any) {
       this._logger.error(`Cannot download differentially, fallback to full download: ${e.stack || e}`)

packages/electron-updater/src/util.ts

@@ -30,8 +30,12 @@ export function getChannelFilename(channel: string): string {
   return `${channel}.yml`
 }
 
-export function blockmapFiles(baseUrl: URL, oldVersion: string, newVersion: string): URL[] {
+export function blockmapFiles(baseUrl: URL, oldVersion: string, newVersion: string, oldBlockMapFileBaseUrl: string | null = null): URL[] {
   const newBlockMapUrl = newUrlFromBase(`${baseUrl.pathname}.blockmap`, baseUrl)
-  const oldBlockMapUrl = newUrlFromBase(`${baseUrl.pathname.replace(new RegExp(escapeRegExp(newVersion), "g"), oldVersion)}.blockmap`, baseUrl)
+  const oldBlockMapUrl = newUrlFromBase(
+    `${baseUrl.pathname.replace(new RegExp(escapeRegExp(newVersion), "g"), oldVersion)}.blockmap`,
+    oldBlockMapFileBaseUrl ? new URL(oldBlockMapFileBaseUrl) : baseUrl
+  )
+
   return [oldBlockMapUrl, newBlockMapUrl]
 }

test/src/binDownloadTest.ts

@@ -17,7 +17,7 @@ test("download binary from Mirror with custom dir", async ({ expect }) => {
     "linux-tools-mac-10.12.3.7z",
     "SQ8fqIRVXuQVWnVgaMTDWyf2TLAJjJYw3tRSqQJECmgF6qdM7Kogfa6KD49RbGzzMYIFca9Uw3MdsxzOPRWcYw=="
   )
-    delete process.env.ELECTRON_BUILDER_BINARIES_MIRROR
+  delete process.env.ELECTRON_BUILDER_BINARIES_MIRROR
   delete process.env.ELECTRON_BUILDER_BINARIES_CUSTOM_DIR
   expect(bin).toBeTruthy()
 })
@@ -29,7 +29,7 @@ test("download binary from Mirror", async ({ expect }) => {
     "linux-tools-mac-10.12.3.7z",
     "SQ8fqIRVXuQVWnVgaMTDWyf2TLAJjJYw3tRSqQJECmgF6qdM7Kogfa6KD49RbGzzMYIFca9Uw3MdsxzOPRWcYw=="
   )
-    delete process.env.ELECTRON_BUILDER_BINARIES_MIRROR
+  delete process.env.ELECTRON_BUILDER_BINARIES_MIRROR
   expect(bin).toBeTruthy()
 })
 
@@ -40,6 +40,6 @@ test("download binary from Mirror with Url override", async ({ expect }) => {
     "linux-tools-mac-10.12.3.7z",
     "SQ8fqIRVXuQVWnVgaMTDWyf2TLAJjJYw3tRSqQJECmgF6qdM7Kogfa6KD49RbGzzMYIFca9Uw3MdsxzOPRWcYw=="
   )
-    delete process.env.ELECTRON_BUILDER_BINARIES_DOWNLOAD_OVERRIDE_URL
+  delete process.env.ELECTRON_BUILDER_BINARIES_DOWNLOAD_OVERRIDE_URL
   expect(bin).toBeTruthy()
 })