Skip to content

Commit 9b18458

Browse files
vis97cvis97c
committed
Merge branch 'dev'
2 parents fdc9bf9 + ec37464 commit 9b18458

File tree

5 files changed

+180
-72
lines changed

5 files changed

+180
-72
lines changed

‎functions/src/types/entities.ts

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,14 @@ export interface FirebaseData extends Record<string, any> {
1919
/**
2020
* Scraped with info
2121
*/
22-
export type ScrapedWith = [eSIALevel, eSIAPlace, uSIAFaculty?, uSIAProgram?, eSIATypology?];
22+
export type ScrapedWith = [
23+
eSIALevel,
24+
eSIAPlace,
25+
uSIAFaculty?,
26+
uSIAProgram?,
27+
eSIATypology?,
28+
uSIAProgram?, // LE program
29+
];
2330

2431
/**
2532
* Firebase log

‎pages/cursos/[courseId].vue

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -602,6 +602,7 @@
602602
// Scrape from old SIA. Do not refetch from hydration
603603
await useFetchQuery<boolean>("/api/groups/scrape", {
604604
code,
605+
options: { credentials: "include" },
605606
});
606607
} catch (err) {
607608
const courseId = <string>route.params.courseId;

‎server/api/groups/scrape.get.ts

Lines changed: 163 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -125,19 +125,6 @@ function useId(id: string): string {
125125
return `#${id.replace(/:/g, "\\:")}`;
126126
}
127127

128-
const puppetConfig: LaunchOptions = {
129-
headless: true,
130-
args: [
131-
"--no-sandbox",
132-
"--disable-setuid-sandbox",
133-
"--disable-dev-shm-usage",
134-
"--disable-accelerated-2d-canvas",
135-
"--no-first-run",
136-
"--no-zygote",
137-
"--disable-gpu",
138-
],
139-
};
140-
141128
/**
142129
* Scrape course from old SIA
143130
*
@@ -181,6 +168,19 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
181168
const updatedByRef = serverFirestore.doc(auth.id);
182169
let attemp = 0;
183170

171+
const puppetConfig: LaunchOptions = {
172+
headless: !debugScrapper,
173+
args: [
174+
"--no-sandbox",
175+
"--disable-setuid-sandbox",
176+
"--disable-dev-shm-usage",
177+
"--disable-accelerated-2d-canvas",
178+
"--no-first-run",
179+
"--no-zygote",
180+
"--disable-gpu",
181+
],
182+
};
183+
184184
// Setup puppeteer
185185
const puppetBrowser: Browser = await launch(puppetConfig);
186186
const puppetPage: Page = await puppetBrowser.newPage();
@@ -392,81 +392,124 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
392392
place: eSIAPlace,
393393
faculties: uSIAFaculty[] = [],
394394
programs: uSIAProgram[] = [],
395-
typologies: eSIATypology[] = []
395+
typologies: eSIATypology[] = [],
396+
programsLE: uSIAProgram[] = [...programs]
396397
): Promise<ScrapedCourse> {
397398
// Compare against multiple sources before giving up
398-
const LE = typologies.includes(eSIATypology.LIBRE_ELECCIÓN);
399-
const LEByProgram = !programs[0];
400399
const LEByFaculty = !faculties[0];
400+
const LEByProgram = !programs[0];
401+
const LEByTypology = typologies.includes(eSIATypology.LIBRE_ELECCIÓN);
401402

402403
// Override data if missing, assume LE
403-
if (LEByProgram || LE || LEByFaculty) {
404+
if (LEByProgram || LEByFaculty || LEByTypology) {
404405
if (!typologies.includes(eSIATypology.LIBRE_ELECCIÓN)) {
405406
typologies.push(eSIATypology.LIBRE_ELECCIÓN);
406407
}
407408

408409
switch (place) {
409410
case eSIAPlace.BOGOTÁ:
410411
if (LEByFaculty) faculties.push(eSIABogotaFaculty.SEDE_BOGOTÁ);
411-
if (LEByProgram || LE) {
412+
if (LEByProgram) {
413+
// Try LE last
412414
programs.push(eSIABogotaProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
413415
}
416+
if (LEByTypology) {
417+
// Try LE first
418+
programsLE.unshift(eSIABogotaProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
419+
}
414420

415421
break;
416422
case eSIAPlace.LA_PAZ:
417423
if (LEByFaculty) faculties.push(eSIALaPazFaculty.SEDE_LA_PAZ);
418-
if (LEByProgram || LE) {
424+
if (LEByProgram) {
425+
// Try LE last
419426
programs.push(eSIALaPazProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
420427
}
428+
if (LEByTypology) {
429+
// Try LE first
430+
programsLE.unshift(eSIALaPazProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
431+
}
421432

422433
break;
423434
case eSIAPlace.MEDELLÍN:
424435
if (LEByFaculty) faculties.push(eSIAMedellinFaculty.SEDE_MEDELLÍN);
425436
if (LEByProgram) {
437+
// Try LE last
426438
programs.push(eSIAMedellinProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
427439
}
440+
if (LEByTypology) {
441+
// Try LE first
442+
programsLE.unshift(eSIAMedellinProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
443+
}
428444

429445
break;
430446
case eSIAPlace.MANIZALES:
431447
if (LEByFaculty) faculties.push(eSIAManizalesFaculty.SEDE_MANIZALES);
432448
if (LEByProgram) {
449+
// Try LE last
433450
programs.push(eSIAManizalesProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
434451
}
452+
if (LEByTypology) {
453+
// Try LE first
454+
programsLE.unshift(eSIAManizalesProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
455+
}
435456

436457
break;
437458
case eSIAPlace.PALMIRA:
438459
if (LEByFaculty) faculties.push(eSIAPalmiraFaculty.SEDE_PALMIRA);
439-
if (LEByProgram || LE) {
440-
programs.push(eSIAPalmiraProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
460+
if (LEByProgram) programs.push(eSIAPalmiraProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
461+
if (LEByTypology) {
462+
// Try LE first
463+
programsLE.unshift(eSIAPalmiraProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
441464
}
442465

443466
break;
444467
case eSIAPlace.TUMACO:
445468
if (LEByFaculty) faculties.push(eSIATumacoFaculty.SEDE_TUMACO);
446-
if (LEByProgram || LE) {
469+
if (LEByProgram) {
470+
// Try LE last
447471
programs.push(eSIATumacoProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
448472
}
473+
if (LEByTypology) {
474+
// Try LE first
475+
programsLE.unshift(eSIATumacoProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
476+
}
449477

450478
break;
451479
case eSIAPlace.AMAZONÍA:
452480
if (LEByFaculty) faculties.push(eSIAAmazoniaFaculty.SEDE_AMAZONIA);
453481
if (LEByProgram) {
482+
// Try LE last
454483
programs.push(eSIAAmazoniaProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
455484
}
485+
if (LEByTypology) {
486+
// Try LE first
487+
programsLE.unshift(eSIAAmazoniaProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
488+
}
456489

457490
break;
458491
case eSIAPlace.CARIBE:
459492
if (LEByFaculty) faculties.push(eSIACaribeFaculty.SEDE_CARIBE);
460-
if (LEByProgram || LE) {
493+
if (LEByProgram) {
494+
// Try LE last
461495
programs.push(eSIACaribeProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
462496
}
497+
if (LEByTypology) {
498+
// Try LE first
499+
programsLE.unshift(eSIACaribeProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
500+
}
463501

464502
break;
465503
case eSIAPlace.ORINOQUÍA:
466504
if (LEByFaculty) faculties.push(eSIAOrinoquiaFaculty.SEDE_ORINOQUIA);
467505
if (LEByProgram) {
506+
// Try LE last
468507
programs.push(eSIAOrinoquiaProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
469508
}
509+
if (LEByTypology) {
510+
// Try LE first
511+
programsLE.unshift(eSIAOrinoquiaProgram.COMPONENTE_DE_LIBRE_ELECCIÓN);
512+
}
470513

471514
break;
472515
}
@@ -565,11 +608,12 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
565608

566609
// Attempt to get groups from this program
567610
if (!typologies.length) {
568-
const { errors = [], ...partialResponse } = await getResponse();
611+
const results = await getResponse();
612+
const { errors = [], ...partial } = results;
569613

570614
response.errors?.push(...errors);
571615
response = {
572-
...partialResponse,
616+
...partial,
573617
lastScrapedWith: [
574618
level,
575619
place,
@@ -612,6 +656,9 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
612656
typologyValue.value
613657
);
614658

659+
// Necessary for switching between typologies
660+
await puppetPage.waitForNetworkIdle();
661+
615662
// Additional actions for LE
616663
if (typology === eSIATypology.LIBRE_ELECCIÓN) {
617664
await puppetPage.waitForSelector(
@@ -632,49 +679,95 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
632679
puppetPage,
633680
eIds.PROGRAM_LE
634681
);
635-
const programLeValue = programOptionsLE.find(
682+
const programLeValues = programOptionsLE.filter(
636683
({ alias }) => {
637684
return (
638685
alias &&
639-
programs.includes(<uSIAProgram>alias)
686+
programsLE.includes(<uSIAProgram>alias)
640687
);
641688
}
642689
);
643690

644-
if (!programLeValue) {
645-
throw new Error("LE program not found");
691+
if (!programLeValues.length) {
692+
throw new Error("LE programs not found");
646693
}
647694

648-
await puppetPage.click(useId(eIds.PROGRAM_LE));
649-
await puppetPage.select(
650-
useId(eIds.PROGRAM_LE),
651-
programLeValue.value
652-
);
695+
// Iterate over associated LE programs
696+
for (const programLeValue of programLeValues) {
697+
if (response.code) break;
698+
699+
debugFirebaseServer(
700+
event,
701+
"api:groups:scrape:scraper:typology:LE",
702+
[
703+
facultyValue.alias,
704+
programValue.alias,
705+
typology,
706+
programLeValue.alias,
707+
],
708+
response.groups?.length
709+
);
710+
711+
try {
712+
// Search by LE program
713+
await puppetPage.click(useId(eIds.PROGRAM_LE));
714+
await puppetPage.select(
715+
useId(eIds.PROGRAM_LE),
716+
programLeValue.value
717+
);
653718

654-
debugFirebaseServer(
655-
event,
656-
"api:groups:scrape:scraper:typology:LE",
657-
programLeValue
658-
);
719+
// Necessary for switching between LE programs
720+
await puppetPage.waitForNetworkIdle();
721+
722+
// Attempt to get groups from this LE program
723+
const results = await getResponse();
724+
const { errors = [], ...partial } = results;
725+
726+
response.errors?.push(...errors);
727+
response = {
728+
...partial,
729+
lastScrapedWith: [
730+
level,
731+
place,
732+
<uSIAFaculty>facultyValue.alias,
733+
<uSIAProgram>programValue.alias,
734+
typology,
735+
<uSIAProgram>programLeValue.alias,
736+
],
737+
};
738+
} catch (err) {
739+
response.errors?.push(err); // save LE programs errors
740+
741+
debugFirebaseServer(
742+
event,
743+
"api:groups:scrape:scraper:typology:LEError",
744+
[
745+
facultyValue.alias,
746+
programValue.alias,
747+
typology,
748+
programLeValue.alias,
749+
],
750+
err
751+
);
752+
}
753+
}
754+
} else {
755+
// Attempt to get groups from this typology
756+
const results = await getResponse();
757+
const { errors = [], ...partial } = results;
758+
759+
response.errors?.push(...errors);
760+
response = {
761+
...partial,
762+
lastScrapedWith: [
763+
level,
764+
place,
765+
<uSIAFaculty>facultyValue.alias,
766+
<uSIAProgram>programValue.alias,
767+
typology,
768+
],
769+
};
659770
}
660-
661-
// Necessary for switching between typologies
662-
await puppetPage.waitForNetworkIdle();
663-
664-
// Attempt to get groups from this typology
665-
const { errors = [], ...partial } = await getResponse();
666-
667-
response.errors?.push(...errors);
668-
response = {
669-
...partial,
670-
lastScrapedWith: [
671-
level,
672-
place,
673-
<uSIAFaculty>facultyValue.alias,
674-
<uSIAProgram>programValue.alias,
675-
typology,
676-
],
677-
};
678771
} catch (err) {
679772
response.errors?.push(err); // save typologies errors
680773

@@ -742,12 +835,20 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
742835
if (!lastScrapedWith) throw new Error("LastScrapedWith is not defined");
743836

744837
// Try last scraped with
745-
const [level, place, faculty, program, typology] = lastScrapedWith;
838+
const [level, place, faculty, program, typology, programLE] = lastScrapedWith;
746839
const faculties = faculty ? [faculty] : [];
747840
const programs = program ? [program] : [];
748841
const typologies = typology ? [typology] : [];
749-
750-
SIAScraps = await scraper(level, place, faculties, programs, typologies);
842+
const programsLE = programLE ? [programLE] : [];
843+
844+
SIAScraps = await scraper(
845+
level,
846+
place,
847+
faculties,
848+
programs,
849+
typologies,
850+
programsLE
851+
);
751852
} catch (err) {
752853
const {
753854
level = eSIALevel.PREGRADO,
@@ -766,7 +867,8 @@ export default defineConditionallyCachedEventHandler(async (event, instance, aut
766867
debugFirebaseServer(
767868
event,
768869
"api:groups:scrape:updateCourse",
769-
`Success scraping: ${code}, ${SIAScraps.name}`
870+
`Success scraping: ${code}, ${SIAScraps.name}`,
871+
SIAScraps.groups
770872
);
771873

772874
// Prevent updating if missing groups

0 commit comments

Comments
 (0)