Add blacklist and other minor changes

README.md

@@ -39,7 +39,7 @@ _Example template_
 
 ## How to use this Github Repository to create your own Hyperlink Aggregator site
 
-### Select your deployment method
+### Select your hosting and deployment method
 
 #### Github Pages
 
@@ -71,9 +71,24 @@ When setting up your own build and continuous integration pipelines, please note
 1. Go to **Secrets and Variables** >> **Actions**
 1. To the repository secrets add the `SUMMARISEAPI` secret
 1. Also add the `SUMMARISEAPIKEY` you copied at step (3)
+1. Go to **Actions** >> **General** and enable Github Actions
+1. Go to **Pages** and for the **Build and deployment** >> **Source** setting, select **Github Actions**
 1. Ensure necessary Pull Request and Commit permissions are set up to allow for post approvals by the administrator and/or moderators
 1. Change the title at the top of this markdown file (`# Post Reader`)
 1. Change the two intro paragraphs at the top of this markdown file, to reflect the topic and purpose of your own website.
 1. In the file located at `src/app/app.config.ts` change the site title on **line 24**
 1. Remove the example posts in the `src/content` folder
+1. See **Base Path Management** below
 1. Submit and commit your first post
+
+### Base Path Management
+
+Due to this repository defaulting to Github Pages, there is a deployment step in the NPM tasks in the [package.json]('./package.json) file to include a base path in the build step. Depending on your hosting selection you will have to make one of the following changes
+
+#### Github Pages (without custom domain)
+
+Go to the [package.json](./package.json) file, line 13, and replace `/post-reader/` with the base path associated with your Github Pages page
+
+#### Github Pages (with custom domain) and others
+
+Depending on your hosting/website configuration, go to the [package.json](./package.json) file and update line 13 as required. Likely you have to remove the `--base-href` argument

package.json

@@ -29,7 +29,7 @@
     "fs": "^0.0.1-security",
     "fs-extra": "^11.2.0",
     "gh-pages": "^6.2.0",
-    "jsdom": "^24.1.0",
+    "jsdom": "^24.1.3",
     "lucide-angular": "^0.411.0",
     "parse-srcset": "^1.0.2",
     "rxjs": "~7.8.0",

src/app/components/ai.component.ts

@@ -1,10 +1,10 @@
-import { ChangeDetectionStrategy, Component } from '@angular/core';
+import { booleanAttribute, ChangeDetectionStrategy, Component, HostBinding, Input } from '@angular/core';
 import { LucideAngularModule } from 'lucide-angular';
 
 @Component({
   selector: 'app-ai',
   template: `<div><ng-content></ng-content></div>
-    <mark>Article summary generated by AI <lucide-icon name="wand" [size]="15" [strokeWidth]="2"></lucide-icon></mark>`,
+    <mark>{{ content }} <lucide-icon name="wand" [size]="15" [strokeWidth]="2"></lucide-icon></mark>`,
   styles: [
     `
       :host {
@@ -16,6 +16,13 @@ import { LucideAngularModule } from 'lucide-angular';
         color: var(--green);
         border-radius: 8px;
         padding: 2px 8px;
+        &.failed {
+          border: 1px solid var(--red);
+          color: var(--red);
+          mark {
+            color: var(--red);
+          }
+        }
       }
 
       div {
@@ -46,4 +53,12 @@ import { LucideAngularModule } from 'lucide-angular';
   imports: [LucideAngularModule],
   standalone: true
 })
-export class AiComponent {}
+export class AiComponent {
+  @HostBinding('class.failed')
+  @Input({ transform: booleanAttribute })
+  failed = false;
+
+  get content(): string {
+    return this.failed ? 'No article summary generated' : 'Article summary generated by AI';
+  }
+}

src/app/components/post/post.component.html

@@ -9,9 +9,7 @@ <h3>{{ post.title }}</h3>
 <div class="meta">
   @if(post.link) {
   <a [href]="post.link" target="_blank">{{ post.link }}</a>
-  }
-
-  @if(post.tags.length) {
+  } @if(post.tags.length) {
   <div class="tags">
     <span>Tags:&nbsp;</span> @for(tag of post.tags; track tag; let i = $index) { @if(i) {, }
     <span class="tag" (click)="tagSelected.emit(tag)">#{{ tag }}</span>
@@ -23,17 +21,24 @@ <h3>{{ post.title }}</h3>
 <ng-container [ngSwitch]="content">
   @if(markdown) {
   <article *ngSwitchCase="'human'" [innerHTML]="markdown"></article>
-  <summary *ngSwitchCase="'machine'">{{ summary }}</summary>
+  <ng-container *ngSwitchCase="'machine'">
+    @if(summary) {
+    <summary>{{ summary }}</summary>
+    } @else {
+    <article [innerHTML]="markdown"></article>
+    <app-ai failed="true" />
+    }
+  </ng-container>
   <ng-container *ngSwitchCase="'both'">
     <article [innerHTML]="markdown"></article>
-    <app-ai>{{ summary }}</app-ai>
+    <app-ai [failed]="!summary">{{ summary }}</app-ai>
   </ng-container>
 
   } @else {
   <summary *ngSwitchCase="'machine'">{{ summary }}</summary>
   <ng-container *ngSwitchDefault>
     <article [innerHTML]="summary"></article>
-    <app-ai></app-ai>
+    <app-ai />
   </ng-container>
   }
-</ng-container>
+</ng-container>

src/bin/models/content-preparation-config.ts

@@ -0,0 +1,3 @@
+export type ContentPreparationConfig = {
+  blacklistScraping: string[];
+};

src/bin/prepare-content-config.ts

@@ -0,0 +1,5 @@
+import { ContentPreparationConfig } from './models/content-preparation-config';
+
+export const PrepareContentConfig: ContentPreparationConfig = {
+  blacklistScraping: []
+};

src/bin/prepare-content.ts

@@ -6,6 +6,7 @@ import { env } from 'process';
 const parseSrcset = require('parse-srcset');
 const jsdom = require('jsdom');
 const { JSDOM } = jsdom;
+import { PrepareContentConfig } from './prepare-content-config';
 require('dotenv').config();
 
 // Function to read all MD files from a directory
@@ -24,7 +25,7 @@ async function readMdFilesFromDirectory(directoryPath: string): Promise<string[]
       }
 
       const [, link] = LinkRegex.exec(markdown) ?? [null, null];
-      if (link) {
+      if (link && PrepareContentConfig.blacklistScraping.every(blacklist => !link.includes(blacklist))) {
         const scraped = await fetch(link).then(response => response.text());
         const { document }: { document: HTMLElement } = new JSDOM(scraped).window;
 
@@ -118,7 +119,8 @@ function enrichWithScrapedData(markdown: string, document: HTMLElement) {
 
   if (scrapedTitle) {
     if (mdTitle) {
-      markdown = markdown.replace(mdTitle, scrapedTitle);
+      // Used to overwrite with scraped title. Caused issues with some scrapes
+      // markdown = markdown.replace(mdTitle, scrapedTitle);
     } else {
       markdown = addElement(markdown, 'Title', scrapedTitle);
     }