Skip to content

Commit fa98a7f

Browse files
cabljaccabljac
committed
refactor: make integration tests more robust
1 parent 32aeadf commit fa98a7f

File tree

4 files changed

+2367
-884
lines changed

4 files changed

+2367
-884
lines changed

‎integration_test/deployment-utils.ts

Lines changed: 276 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,276 @@
1+
import pRetry from "p-retry";
2+
import pLimit from "p-limit";
3+
4+
interface FirebaseClient {
5+
functions: {
6+
list: () => Promise<{ name: string }[]>;
7+
delete(names: string[], options: any): Promise<void>;
8+
};
9+
deploy: (options: { only: string; force: boolean }) => Promise<void>;
10+
}
11+
12+
// Configuration constants
13+
const BATCH_SIZE = 3; // Reduced to 3 functions at a time for better rate limiting
14+
const DELAY_BETWEEN_BATCHES = 5000; // Increased from 2 to 5 seconds between batches
15+
const MAX_RETRIES = 3; // Retry failed deployments
16+
const CLEANUP_DELAY = 1000; // 1 second between cleanup operations
17+
// Rate limiter for deployment operations
18+
const deploymentLimiter = pLimit(1); // Only one deployment operation at a time
19+
const cleanupLimiter = pLimit(2); // Allow 2 cleanup operations concurrently
20+
21+
/**
22+
* Sleep utility function
23+
*/
24+
const sleep = (ms: number): Promise<void> => new Promise((resolve) => setTimeout(resolve, ms));
25+
26+
/**
27+
* Get all deployed functions for the current project
28+
*/
29+
export async function getDeployedFunctions(client: FirebaseClient): Promise<string[]> {
30+
try {
31+
const functions = await client.functions.list();
32+
return functions.map((fn: { name: string }) => fn.name);
33+
} catch (error) {
34+
console.log("Could not list functions, assuming none deployed:", error);
35+
return [];
36+
}
37+
}
38+
39+
/**
40+
* Delete a single function with retry logic
41+
*/
42+
async function deleteFunctionWithRetry(
43+
client: FirebaseClient,
44+
functionName: string
45+
): Promise<void> {
46+
return pRetry(
47+
async () => {
48+
try {
49+
await client.functions.delete([functionName], {
50+
force: true,
51+
project: process.env.PROJECT_ID,
52+
config: "./firebase.json",
53+
debug: true,
54+
nonInteractive: true,
55+
});
56+
console.log(`✅ Deleted function: ${functionName}`);
57+
} catch (error: unknown) {
58+
if (
59+
error &&
60+
typeof error === "object" &&
61+
"message" in error &&
62+
typeof error.message === "string" &&
63+
error.message.includes("not found")
64+
) {
65+
console.log(`ℹ️ Function not found (already deleted): ${functionName}`);
66+
return; // Not an error, function was already deleted
67+
}
68+
throw error;
69+
}
70+
},
71+
{
72+
retries: MAX_RETRIES,
73+
onFailedAttempt: (error) => {
74+
console.log(
75+
`❌ Failed to delete ${functionName} (attempt ${error.attemptNumber}/${
76+
MAX_RETRIES + 1
77+
}):`,
78+
error.message
79+
);
80+
},
81+
}
82+
);
83+
}
84+
85+
/**
86+
* Pre-cleanup: Remove all existing functions before deployment
87+
*/
88+
export async function preCleanup(client: FirebaseClient): Promise<void> {
89+
console.log("🧹 Starting pre-cleanup...");
90+
91+
try {
92+
const deployedFunctions = await getDeployedFunctions(client);
93+
94+
if (deployedFunctions.length === 0) {
95+
console.log("ℹ️ No functions to clean up");
96+
return;
97+
}
98+
99+
console.log(`Found ${deployedFunctions.length} functions to clean up`);
100+
101+
// Delete functions in batches with rate limiting
102+
const batches: string[][] = [];
103+
for (let i = 0; i < deployedFunctions.length; i += BATCH_SIZE) {
104+
batches.push(deployedFunctions.slice(i, i + BATCH_SIZE));
105+
}
106+
107+
for (let i = 0; i < batches.length; i++) {
108+
const batch = batches[i];
109+
console.log(`Cleaning up batch ${i + 1}/${batches.length} (${batch.length} functions)`);
110+
111+
// Delete functions in parallel within the batch
112+
const deletePromises = batch.map((functionName) =>
113+
cleanupLimiter(() => deleteFunctionWithRetry(client, functionName))
114+
);
115+
116+
await Promise.all(deletePromises);
117+
118+
// Add delay between batches
119+
if (i < batches.length - 1) {
120+
console.log(`Waiting ${CLEANUP_DELAY}ms before next batch...`);
121+
await sleep(CLEANUP_DELAY);
122+
}
123+
}
124+
125+
console.log("✅ Pre-cleanup completed");
126+
} catch (error) {
127+
console.error("❌ Pre-cleanup failed:", error);
128+
throw error;
129+
}
130+
}
131+
132+
/**
133+
* Deploy functions with rate limiting and retry logic
134+
*/
135+
export async function deployFunctionsWithRetry(
136+
client: any,
137+
functionsToDeploy: string[]
138+
): Promise<void> {
139+
console.log(`🚀 Deploying ${functionsToDeploy.length} functions with rate limiting...`);
140+
141+
// Deploy functions in batches
142+
const batches = [];
143+
for (let i = 0; i < functionsToDeploy.length; i += BATCH_SIZE) {
144+
batches.push(functionsToDeploy.slice(i, i + BATCH_SIZE));
145+
}
146+
147+
for (let i = 0; i < batches.length; i++) {
148+
const batch = batches[i];
149+
console.log(`Deploying batch ${i + 1}/${batches.length} (${batch.length} functions)`);
150+
151+
try {
152+
await pRetry(
153+
async () => {
154+
await deploymentLimiter(async () => {
155+
await client.deploy({
156+
only: "functions",
157+
force: true,
158+
});
159+
});
160+
},
161+
{
162+
retries: MAX_RETRIES,
163+
onFailedAttempt: (error: any) => {
164+
console.log(
165+
`❌ Deployment failed (attempt ${error.attemptNumber}/${MAX_RETRIES + 1}):`,
166+
error.message
167+
);
168+
// Log detailed error information during retries
169+
if (error.children && error.children.length > 0) {
170+
console.log("📋 Detailed deployment errors:");
171+
error.children.forEach((child: any, index: number) => {
172+
console.log(` ${index + 1}. ${child.message || child}`);
173+
if (child.original) {
174+
console.log(
175+
` Original error message: ${child.original.message || "No message"}`
176+
);
177+
console.log(` Original error code: ${child.original.code || "No code"}`);
178+
console.log(
179+
` Original error status: ${child.original.status || "No status"}`
180+
);
181+
}
182+
});
183+
}
184+
// Log the full error structure for debugging
185+
console.log("🔍 Error details:");
186+
console.log(` - Message: ${error.message}`);
187+
console.log(` - Status: ${error.status}`);
188+
console.log(` - Exit code: ${error.exit}`);
189+
console.log(` - Attempt: ${error.attemptNumber}`);
190+
console.log(` - Retries left: ${error.retriesLeft}`);
191+
},
192+
}
193+
);
194+
195+
console.log(`✅ Batch ${i + 1} deployed successfully`);
196+
197+
// Add delay between batches
198+
if (i < batches.length - 1) {
199+
console.log(`Waiting ${DELAY_BETWEEN_BATCHES}ms before next batch...`);
200+
await sleep(DELAY_BETWEEN_BATCHES);
201+
}
202+
} catch (error: any) {
203+
console.error(`❌ Failed to deploy batch ${i + 1}:`, error);
204+
// Log detailed error information
205+
if (error.children && error.children.length > 0) {
206+
console.log("📋 Detailed deployment errors:");
207+
error.children.forEach((child: any, index: number) => {
208+
console.log(` ${index + 1}. ${child.message || child}`);
209+
if (child.original) {
210+
console.log(` Original error message: ${child.original.message || "No message"}`);
211+
console.log(` Original error code: ${child.original.code || "No code"}`);
212+
console.log(` Original error status: ${child.original.status || "No status"}`);
213+
}
214+
});
215+
}
216+
// Log the full error structure for debugging
217+
console.log("🔍 Error details:");
218+
console.log(` - Message: ${error.message}`);
219+
console.log(` - Status: ${error.status}`);
220+
console.log(` - Exit code: ${error.exit}`);
221+
console.log(` - Attempt: ${error.attemptNumber}`);
222+
console.log(` - Retries left: ${error.retriesLeft}`);
223+
throw error;
224+
}
225+
}
226+
227+
console.log("✅ All functions deployed successfully");
228+
}
229+
230+
/**
231+
* Post-cleanup: Remove deployed functions after tests
232+
*/
233+
export async function postCleanup(client: any, testRunId: string): Promise<void> {
234+
console.log("🧹 Starting post-cleanup...");
235+
236+
try {
237+
const deployedFunctions = await getDeployedFunctions(client);
238+
const testFunctions = deployedFunctions.filter((name) => name && name.includes(testRunId));
239+
240+
if (testFunctions.length === 0) {
241+
console.log("ℹ️ No test functions to clean up");
242+
return;
243+
}
244+
245+
console.log(`Found ${testFunctions.length} test functions to clean up`);
246+
247+
// Delete test functions in batches with rate limiting
248+
const batches = [];
249+
for (let i = 0; i < testFunctions.length; i += BATCH_SIZE) {
250+
batches.push(testFunctions.slice(i, i + BATCH_SIZE));
251+
}
252+
253+
for (let i = 0; i < batches.length; i++) {
254+
const batch = batches[i];
255+
console.log(`Cleaning up batch ${i + 1}/${batches.length} (${batch.length} functions)`);
256+
257+
// Delete functions in parallel within the batch
258+
const deletePromises = batch.map((functionName) =>
259+
cleanupLimiter(() => deleteFunctionWithRetry(client, functionName))
260+
);
261+
262+
await Promise.all(deletePromises);
263+
264+
// Add delay between batches
265+
if (i < batches.length - 1) {
266+
console.log(`Waiting ${CLEANUP_DELAY}ms before next batch...`);
267+
await sleep(CLEANUP_DELAY);
268+
}
269+
}
270+
271+
console.log("✅ Post-cleanup completed");
272+
} catch (error) {
273+
console.error("❌ Post-cleanup failed:", error);
274+
throw error;
275+
}
276+
}

0 commit comments

Comments
 (0)