fix: links can now be imported even if they are uncreachable.

Author: RobRoyce

src/kc_angular/src/app/components/shared/create.component.ts

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022 Rob Royce
+ * Copyright (c) 2022-2023 Rob Royce
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ import {DragAndDropService} from "../../services/ingest-services/drag-and-drop.s
 import {FaviconService} from "../../services/ingest-services/favicon.service";
 import {HttpClient} from "@angular/common/http";
 import {DomSanitizer} from "@angular/platform-browser";
+import {ConfirmationService} from "primeng/api";
 
 @Component({
   selector: 'app-create',
@@ -94,6 +95,7 @@ export class CreateComponent implements OnInit {
   files: any[] = [];
 
   constructor(private dialog: DialogService,
+              private confirm: ConfirmationService,
               private factory: KsFactoryService,
               private ingest: IngestService,
               private notifications: NotificationsService,
@@ -146,17 +148,63 @@ export class CreateComponent implements OnInit {
 
       this.factory.many(req).then((ksList) => {
         if (!ksList || !ksList.length) {
-          this.notifications.error('Source Import', 'Unable to import URL', value);
+          this.notifications.error('Source Import', `Unable to import URL`, value);
           return;
         }
 
         this.ingest.enqueue(ksList);
 
-      }).catch((_) => {
-        this.notifications.error('Source Import', 'Unable to import URL', value);
+      }).catch((error) => {
+        this.notifications.error('Source Import', `Unable to import URL`, `Source Factory: ${value}`);
+      });
+    }, (error) => {
+      this.notifications.error('Source Import', `Invalid URL`, `HTTP Client: ${value}`);
+      console.log(error)
+
+      // If the link cannot be reached, ask the user if they still want to import it anyway
+      this.confirm.confirm({
+        header: "Link Unreachable",
+        message: `
+        <div>
+            <div style="max-width: 12rem;">
+                ${url.href} is unreachable.
+            </div>
+            <br>
+            <div style="max-width: 12rem;">
+                Sometimes this happens when the website is behind a paywall, requires authentication, or expects certain cookies.
+            </div>
+            <br>
+            <div>
+                  You can still add it to Knowledge and we will attempt to extract information from it later on.
+            </div>
+            <br>
+            <div>
+                  Do you want to import it anyway?
+            </div>
+        </div>
+        `,
+        icon: 'pi pi-warn',
+        acceptLabel: 'Import',
+        rejectLabel: 'Cancel',
+        accept: () => {
+          let req: KnowledgeSourceFactoryRequest = {
+            ingestType: 'website',
+            links: [url]
+          }
+
+          this.factory.many(req).then((ksList) => {
+            if (!ksList || !ksList.length) {
+              this.notifications.error('Source Import', `Unable to import URL`, value);
+              return;
+            }
+
+            this.ingest.enqueue(ksList);
+
+          }).catch((error) => {
+            this.notifications.error('Source Import', `Unable to import URL`, `Source Factory: ${value}`);
+          });
+        }
       });
-    }, (_) => {
-      this.notifications.error('Source Import', 'Invalid URL', value);
       return;
     });
 

src/kc_angular/src/app/services/ingest-services/extractor.service.ts

@@ -1,22 +1,27 @@
-/**
- Copyright 2022 Rob Royce
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
+/*
+ * Copyright (c) 2023 Rob Royce
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
  */
 import {Injectable} from '@angular/core';
 import {HttpClient} from "@angular/common/http";
 import {NotificationsService} from "../user-services/notifications.service";
-import {ArticleModel, CodeModel, WebsiteMetadataModel, WebsiteMetaTagsModel} from "../../../../../kc_shared/models/web.source.model";
+import {
+  ArticleModel,
+  CodeModel,
+  WebsiteMetadataModel,
+  WebsiteMetaTagsModel
+} from "../../../../../kc_shared/models/web.source.model";
 
 @Injectable({
   providedIn: 'root'
@@ -175,6 +180,10 @@ export class ExtractorService {
         // Meta Tags
         metadata.meta = extractMetaTags(htmlDoc);
 
+        resolve(metadata);
+      }, (error) => {
+        this.notifications.error('ExtractorService', 'extractWebsiteMetadata', 'Error retrieving website.');
+        metadata.title = url;
         resolve(metadata);
       });
     });