Skip to content

Commit 02b1f3f

Browse files
alexander-akaitalexander-akait
authored
fix: base64 and unicode characters
1 parent a30b4b7 commit 02b1f3f

File tree

7 files changed

+240
-31
lines changed

7 files changed

+240
-31
lines changed

‎.github/workflows/nodejs.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ jobs:
7474
run: yarn
7575

7676
- name: Run tests
77-
run: yarn test
77+
run: yarn test:only
7878

7979
- name: Submit coverage data to codecov
8080
uses: codecov/codecov-action@v2

‎lib/getHashDigest.js‎

Lines changed: 27 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,37 +45,54 @@ function encodeBufferToBase(buffer, base) {
4545
let crypto = undefined;
4646
let createXXHash64 = undefined;
4747
let createMd4 = undefined;
48+
let BatchedHash = undefined;
49+
let BulkUpdateDecorator = undefined;
4850

49-
function getHashDigest(buffer, hashType, digestType, maxLength) {
50-
hashType = hashType || "xxhash64";
51+
function getHashDigest(buffer, algorithm, digestType, maxLength) {
52+
algorithm = algorithm || "xxhash64";
5153
maxLength = maxLength || 9999;
5254

5355
let hash;
5456

55-
if (hashType === "xxhash64") {
57+
if (algorithm === "xxhash64") {
5658
if (createXXHash64 === undefined) {
5759
createXXHash64 = require("./hash/xxhash64");
60+
61+
if (BatchedHash === undefined) {
62+
BatchedHash = require("./hash/BatchedHash");
63+
}
5864
}
5965

60-
hash = createXXHash64();
61-
} else if (hashType === "md4") {
66+
hash = new BatchedHash(createXXHash64());
67+
} else if (algorithm === "md4") {
6268
if (createMd4 === undefined) {
6369
createMd4 = require("./hash/md4");
6470
}
6571

66-
hash = createMd4();
67-
} else if (hashType === "native-md4") {
72+
hash = new BatchedHash(createMd4());
73+
} else if (algorithm === "native-md4") {
6874
if (typeof crypto === "undefined") {
6975
crypto = require("crypto");
76+
77+
if (BulkUpdateDecorator === undefined) {
78+
BulkUpdateDecorator = require("./hash/BulkUpdateDecorator");
79+
}
7080
}
7181

72-
hash = crypto.createHash("md4");
82+
hash = new BulkUpdateDecorator(() => crypto.createHash("md4"), "md4");
7383
} else {
7484
if (typeof crypto === "undefined") {
7585
crypto = require("crypto");
86+
87+
if (BulkUpdateDecorator === undefined) {
88+
BulkUpdateDecorator = require("./hash/BulkUpdateDecorator");
89+
}
7690
}
7791

78-
hash = crypto.createHash(hashType);
92+
hash = new BulkUpdateDecorator(
93+
() => crypto.createHash(algorithm),
94+
algorithm
95+
);
7996
}
8097

8198
hash.update(buffer);
@@ -87,8 +104,7 @@ function getHashDigest(buffer, hashType, digestType, maxLength) {
87104
digestType === "base49" ||
88105
digestType === "base52" ||
89106
digestType === "base58" ||
90-
digestType === "base62" ||
91-
digestType === "base64"
107+
digestType === "base62"
92108
) {
93109
return encodeBufferToBase(hash.digest(), digestType.substr(4)).substr(
94110
0,

‎lib/hash/BatchedHash.js‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
const MAX_SHORT_STRING = require("./wasm-hash").MAX_SHORT_STRING;
2+
3+
class BatchedHash {
4+
constructor(hash) {
5+
this.string = undefined;
6+
this.encoding = undefined;
7+
this.hash = hash;
8+
}
9+
10+
/**
11+
* Update hash {@link https://nodejs.org/api/crypto.html#crypto_hash_update_data_inputencoding}
12+
* @param {string|Buffer} data data
13+
* @param {string=} inputEncoding data encoding
14+
* @returns {this} updated hash
15+
*/
16+
update(data, inputEncoding) {
17+
if (this.string !== undefined) {
18+
if (
19+
typeof data === "string" &&
20+
inputEncoding === this.encoding &&
21+
this.string.length + data.length < MAX_SHORT_STRING
22+
) {
23+
this.string += data;
24+
25+
return this;
26+
}
27+
28+
this.hash.update(this.string, this.encoding);
29+
this.string = undefined;
30+
}
31+
32+
if (typeof data === "string") {
33+
if (
34+
data.length < MAX_SHORT_STRING &&
35+
// base64 encoding is not valid since it may contain padding chars
36+
(!inputEncoding || !inputEncoding.startsWith("ba"))
37+
) {
38+
this.string = data;
39+
this.encoding = inputEncoding;
40+
} else {
41+
this.hash.update(data, inputEncoding);
42+
}
43+
} else {
44+
this.hash.update(data);
45+
}
46+
47+
return this;
48+
}
49+
50+
/**
51+
* Calculates the digest {@link https://nodejs.org/api/crypto.html#crypto_hash_digest_encoding}
52+
* @param {string=} encoding encoding of the return value
53+
* @returns {string|Buffer} digest
54+
*/
55+
digest(encoding) {
56+
if (this.string !== undefined) {
57+
this.hash.update(this.string, this.encoding);
58+
}
59+
60+
return this.hash.digest(encoding);
61+
}
62+
}
63+
64+
module.exports = BatchedHash;

‎lib/hash/BulkUpdateDecorator.js‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
const BULK_SIZE = 2000;
2+
3+
// We are using an object instead of a Map as this will stay static during the runtime
4+
// so access to it can be optimized by v8
5+
const digestCaches = {};
6+
7+
class BulkUpdateDecorator {
8+
/**
9+
* @param {Hash | function(): Hash} hashOrFactory function to create a hash
10+
* @param {string=} hashKey key for caching
11+
*/
12+
constructor(hashOrFactory, hashKey) {
13+
this.hashKey = hashKey;
14+
15+
if (typeof hashOrFactory === "function") {
16+
this.hashFactory = hashOrFactory;
17+
this.hash = undefined;
18+
} else {
19+
this.hashFactory = undefined;
20+
this.hash = hashOrFactory;
21+
}
22+
23+
this.buffer = "";
24+
}
25+
26+
/**
27+
* Update hash {@link https://nodejs.org/api/crypto.html#crypto_hash_update_data_inputencoding}
28+
* @param {string|Buffer} data data
29+
* @param {string=} inputEncoding data encoding
30+
* @returns {this} updated hash
31+
*/
32+
update(data, inputEncoding) {
33+
if (
34+
inputEncoding !== undefined ||
35+
typeof data !== "string" ||
36+
data.length > BULK_SIZE
37+
) {
38+
if (this.hash === undefined) {
39+
this.hash = this.hashFactory();
40+
}
41+
42+
if (this.buffer.length > 0) {
43+
this.hash.update(this.buffer);
44+
this.buffer = "";
45+
}
46+
47+
this.hash.update(data, inputEncoding);
48+
} else {
49+
this.buffer += data;
50+
51+
if (this.buffer.length > BULK_SIZE) {
52+
if (this.hash === undefined) {
53+
this.hash = this.hashFactory();
54+
}
55+
56+
this.hash.update(this.buffer);
57+
this.buffer = "";
58+
}
59+
}
60+
61+
return this;
62+
}
63+
64+
/**
65+
* Calculates the digest {@link https://nodejs.org/api/crypto.html#crypto_hash_digest_encoding}
66+
* @param {string=} encoding encoding of the return value
67+
* @returns {string|Buffer} digest
68+
*/
69+
digest(encoding) {
70+
let digestCache;
71+
72+
const buffer = this.buffer;
73+
74+
if (this.hash === undefined) {
75+
// short data for hash, we can use caching
76+
const cacheKey = `${this.hashKey}-${encoding}`;
77+
78+
digestCache = digestCaches[cacheKey];
79+
80+
if (digestCache === undefined) {
81+
digestCache = digestCaches[cacheKey] = new Map();
82+
}
83+
84+
const cacheEntry = digestCache.get(buffer);
85+
86+
if (cacheEntry !== undefined) {
87+
return cacheEntry;
88+
}
89+
90+
this.hash = this.hashFactory();
91+
}
92+
93+
if (buffer.length > 0) {
94+
this.hash.update(buffer);
95+
}
96+
97+
const digestResult = this.hash.digest(encoding);
98+
99+
if (digestCache !== undefined) {
100+
digestCache.set(buffer, digestResult);
101+
}
102+
103+
return digestResult;
104+
}
105+
}
106+
107+
module.exports = BulkUpdateDecorator;

‎package.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,11 @@
77
"big.js": "^6.1.1"
88
},
99
"scripts": {
10-
"lint": "prettier --list-different . && eslint lib test",
10+
"lint": "prettier --list-different . && eslint .",
1111
"pretest": "yarn lint",
1212
"test": "jest",
13-
"test:ci": "jest --coverage",
13+
"test:only": "jest --coverage",
14+
"test:ci": "yarn test:only",
1415
"release": "yarn test && standard-version"
1516
},
1617
"license": "MIT",

‎test/getHashDigest.test.js‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,36 +4,57 @@ const loaderUtils = require("../");
44

55
describe("getHashDigest()", () => {
66
[
7+
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
8+
["test string", "xxhash64", "base64", undefined, "6eLDUePGsZg="],
9+
["test string", "xxhash64", "base52", undefined, "byfYGDmnmyUr"],
10+
["abc\\0♥", "xxhash64", "hex", undefined, "4b9a34297dc03d20"],
11+
["abc\\0💩", "xxhash64", "hex", undefined, "86733ec125b93904"],
12+
["abc\\0💩", "xxhash64", "base64", undefined, "hnM+wSW5OQQ="],
13+
["abc\\0♥", "xxhash64", "base64", undefined, "S5o0KX3APSA="],
14+
["abc\\0💩", "xxhash64", "base52", undefined, "cfByjQcJZIU"],
15+
["abc\\0♥", "xxhash64", "base52", undefined, "qdLyAQjLlod"],
16+
17+
["test string", "md4", "hex", 4, "2e06"],
18+
["test string", "md4", "base64", undefined, "Lgbt1PFiMmjFpRcw2KCyrw=="],
19+
["test string", "md4", "base52", undefined, "egWqIKxsDHdZTteemJqXfuo"],
20+
["abc\\0♥", "md4", "hex", undefined, "46b9627fecf49b80eaf01c01d86ae9fd"],
21+
["abc\\0💩", "md4", "hex", undefined, "45aa5b332f8e562aaf0106ad6fc1d78f"],
22+
["abc\\0💩", "md4", "base64", undefined, "RapbMy+OViqvAQatb8HXjw=="],
23+
["abc\\0♥", "md4", "base64", undefined, "Rrlif+z0m4Dq8BwB2Grp/Q=="],
24+
["abc\\0💩", "md4", "base52", undefined, "dtXZENFEkYHXGxOkJbevPoD"],
25+
["abc\\0♥", "md4", "base52", undefined, "fYFFcfXRGsVweukHKlPayHs"],
26+
27+
["test string", "md5", "hex", 4, "6f8d"],
728
[
829
"test string",
930
"md5",
1031
"hex",
1132
undefined,
1233
"6f8db599de986fab7a21625b7916589c",
1334
],
14-
["test string", "md5", "base64", undefined, "2sm1pVmS8xuGJLCdWpJoRL"],
15-
// ["test string", "md5", "base64url", undefined, "b421md6Yb6t6IWJbeRZYnA"],
16-
["test string", "xxhash64", "hex", undefined, "e9e2c351e3c6b198"],
17-
["test string", "xxhash64", "base64", undefined, "9yNNKdhM-bF"],
18-
["test string", "xxhash64", "base52", undefined, "byfYGDmnmyUr"],
19-
// ["test string", "xxhash64", "base64url", undefined, "6eLDUePGsZg"],
20-
["test string", "md4", "hex", 4, "2e06"],
21-
["test string", "md5", "hex", 4, "6f8d"],
2235
["test string", "md5", "base52", undefined, "dJnldHSAutqUacjgfBQGLQx"],
36+
["test string", "md5", "base64", undefined, "b421md6Yb6t6IWJbeRZYnA=="],
2337
["test string", "md5", "base26", 6, "bhtsgu"],
38+
["abc\\0♥", "md5", "hex", undefined, "2e897b64f8050e66aff98d38f7a012c5"],
39+
["abc\\0💩", "md5", "hex", undefined, "63ad5b3d675c5890e0c01ed339ba0187"],
40+
["abc\\0💩", "md5", "base64", undefined, "Y61bPWdcWJDgwB7TOboBhw=="],
41+
["abc\\0♥", "md5", "base64", undefined, "Lol7ZPgFDmav+Y0496ASxQ=="],
42+
["abc\\0💩", "md5", "base52", undefined, "djhVWGHaUKUxqxEhcTnOfBx"],
43+
["abc\\0♥", "md5", "base52", undefined, "eHeasSeRyOnorzxUJpayzJc"],
44+
2445
[
2546
"test string",
2647
"sha512",
2748
"base64",
2849
undefined,
29-
"2IS-kbfIPnVflXb9CzgoNESGCkvkb0urMmucPD9z8q6HuYz8RShY1-tzSUpm5-Ivx_u4H1MEzPgAhyhaZ7RKog",
50+
"EObWR69EYkRC84jCwUp4f/ixfmFluD12fsBHdo2MvLcaGjIm58x4Frx5wEJ9lKnaaIxBo5kse/Xk18w+C+XbrA==",
3051
],
3152
[
3253
"test string",
33-
"md5",
54+
"sha512",
3455
"hex",
3556
undefined,
36-
"6f8db599de986fab7a21625b7916589c",
57+
"10e6d647af44624442f388c2c14a787ff8b17e6165b83d767ec047768d8cbcb71a1a3226e7cc7816bc79c0427d94a9da688c41a3992c7bf5e4d7cc3e0be5dbac",
3758
],
3859
].forEach((test) => {
3960
it(

‎test/interpolateName.test.js‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,13 @@ describe("interpolateName()", () => {
5050
"/app/img/image.png",
5151
"[sha512:hash:base64:7].[ext]",
5252
"test content",
53-
"2BKDTjl.png",
53+
"DL9MrvO.png",
5454
],
5555
[
5656
"/app/img/image.png",
5757
"[sha512:contenthash:base64:7].[ext]",
5858
"test content",
59-
"2BKDTjl.png",
59+
"DL9MrvO.png",
6060
],
6161
[
6262
"/app/dir/file.png",
@@ -104,19 +104,19 @@ describe("interpolateName()", () => {
104104
"/lib/components/modal/modal.css",
105105
"[name].[md4:hash:base64:20].[ext]",
106106
"test content",
107-
"modal.1kNSGJ6n9ibMUEckC1Cp.css",
107+
"modal.ppiZgUkxKA4vUnIZrWrH.css",
108108
],
109109
[
110110
"/lib/components/modal/modal.css",
111111
"[name].[md5:hash:base64:20].[ext]",
112112
"test content",
113-
"modal.1n8osQznuT8jOAwdzg_n.css",
113+
"modal.lHP90NiApDwht3eNNIch.css",
114114
],
115115
[
116116
"/lib/components/modal/modal.css",
117117
"[name].[md5:contenthash:base64:20].[ext]",
118118
"test content",
119-
"modal.1n8osQznuT8jOAwdzg_n.css",
119+
"modal.lHP90NiApDwht3eNNIch.css",
120120
],
121121
// Should not interpret without `hash` or `contenthash`
122122
[
@@ -259,7 +259,7 @@ describe("interpolateName()", () => {
259259
],
260260
[
261261
[{}, "[hash:base64]", { content: "test string" }],
262-
"9yNNKdhM-bF",
262+
"6eLDUePGsZg=",
263263
"should interpolate [hash] token with options",
264264
],
265265
[

0 commit comments

Comments
 (0)