First Commit

.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+.env
+logs/
+db.json
+lib/

package.json

@@ -0,0 +1,35 @@
+{
+    "name": "rss-telegram-bot",
+    "version": "1.0.0",
+    "main": "lib/index.js",
+    "author": "Fabian Stamm <dev@fabianstamm.de>",
+    "license": "MIT",
+    "private": true,
+    "scripts": {
+        "start": "node lib/index.js",
+        "build": "tsc",
+        "watch-ts": "tsc -w",
+        "watch-js": "nodemon -i db.json lib/index.js",
+        "watch": "concurrently npm:watch-*"
+    },
+    "devDependencies": {
+        "@types/dotenv": "^4.0.3",
+        "@types/html-entities": "^1.2.16",
+        "@types/lowdb": "^1.0.5",
+        "@types/node": "^10.7.0",
+        "@types/node-fetch": "^2.1.2",
+        "concurrently": "^4.1.0",
+        "nodemon": "^1.18.3",
+        "typescript": "^3.0.1"
+    },
+    "dependencies": {
+        "@hibas123/nodelogging": "^1.3.12",
+        "dotenv": "^6.0.0",
+        "html-entities": "^1.2.1",
+        "lowdb": "^1.0.0",
+        "node-fetch": "^2.2.0",
+        "rss-parser": "^3.4.2",
+        "ssl-root-cas": "^1.2.5",
+        "telegraf": "^3.24.1"
+    }
+}

src/index.ts

@@ -0,0 +1,218 @@
+require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create();
+require('ssl-root-cas').inject();
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
+
+import Logging from "@hibas123/nodelogging";
+import { createHash } from "crypto";
+import * as dotenv from "dotenv";
+import { AllHtmlEntities } from "html-entities";
+import low from "lowdb";
+import FileSync from "lowdb/adapters/FileSync";
+import fetch from "node-fetch";
+import rss from "rss-parser";
+import Telegraf from "telegraf";
+
+dotenv.config();
+const parser = new rss();
+const entities = new AllHtmlEntities();
+
+const adapter = new FileSync<{ feeds: IDBFeed[] }>("db.json");
+const db = low(adapter);
+db.defaults({ feeds: [] }).write();
+
+interface IDBFeed {
+    url: string;
+    subscriber: number[];
+    oldEntries: string[];
+}
+
+class Database {
+    static findFeed(url: string): DBFeed {
+        const feed = db.get("feeds").find(e => e.url === url).value();
+        return feed ? new DBFeed(feed) : undefined;
+    }
+
+
+    static addFeed(url: string): IDBFeed {
+        const feed = {
+            url: url,
+            oldEntries: [],
+            subscriber: []
+        };
+        db.get("feeds").unshift(feed).write();
+        return feed;
+    }
+
+    static addSubscriber(url: string, chatid: number) {
+        const feed = this.findFeed(url);
+        if (!feed)
+            this.addFeed(url);
+        else {
+            if (feed.subscriber.some(e => e === chatid)) {
+                return;
+            }
+        }
+        db.get("feeds").find(e => e.url === url).get("subscriber").push(chatid).write();
+    }
+
+    static findSubscribed(chatid: number) {
+        return db.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value();
+    }
+
+    static removeSubscriber(url: string, chatid: number) {
+        db.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid).write();
+    }
+
+    static addItems(url: string, hashes: string[]) {
+        db.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes).write();
+    }
+
+    static getAll() {
+        return db.get("feeds").map(feed => new DBFeed(feed)).value();
+    }
+}
+
+
+class DBFeed implements IDBFeed {
+    url: string;
+    subscriber: number[];
+    oldEntries: string[];
+
+    constructor(dbobject?: IDBFeed) {
+        if (dbobject) {
+            for (let key in dbobject)
+                this[key] = dbobject[key];
+        }
+    }
+
+    itemExists(item: IFeedItem): boolean {
+        const hash = calculateHash(item);
+        return !!this.oldEntries.find(e => e === hash);
+    }
+
+    addItems(items: IFeedItem[]) {
+        Database.addItems(this.url, items.map(item => calculateHash(item)));
+    }
+}
+
+
+interface Feed {
+    title: string;
+    language: string;
+    description: string;
+    copyright: string;
+
+    items: IFeedItem[];
+}
+
+interface IFeedItem {
+    title: string;
+    link: string;
+    content: string,
+    contentSnippet: string;
+    guid: string;
+}
+
+function calculateHash(item: IFeedItem) {
+    let hash = createHash("sha512");
+    if (item.content)
+        hash.update(item.content);
+    if (item.guid)
+        hash.update(item.guid);
+    if (item.title)
+        hash.update(item.title);
+    if (item.link)
+        hash.update(item.link);
+    if (item.contentSnippet)
+        hash.update(item.contentSnippet);
+    return hash.digest("hex");
+}
+
+async function checkFeed(feed: DBFeed) {
+    Logging.log("Fetching:", feed.url);
+    let data = await fetch(feed.url).then(res => res.text());
+    Logging.log("Received Data");
+
+    let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed;
+
+    // Check for new items
+    Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
+    const newItems = feedData.items.filter(item => !feed.itemExists(item));
+
+    // Sending notifications
+    await sendFeedTelegraf(feed, newItems);
+
+    feed.addItems(newItems);
+}
+
+
+const bot = new Telegraf(process.env.TG_TOKEN)
+
+bot.start(async ctx => {
+    await ctx.reply("Send some RSS Feed URLs to get started.");
+})
+
+bot.command("list", async (ctx) => {
+    const chatid = ctx.chat.id;
+    const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n");
+    await ctx.reply("You are currently subscribed to: \n" + feeds);
+})
+
+bot.on("message", async ctx => {
+    const chatid = ctx.chat.id;
+
+    const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length));
+    Logging.debug(ctx.message);
+    await Promise.all(urls.map(async url => {
+        Database.addSubscriber(url, chatid);
+        await ctx.reply("Subscribed to: " + url);
+        let feed = Database.findFeed(url)
+        if (feed)
+            checkFeed(feed);
+        else
+            Logging.error("Cannot find created feed!")
+    }))
+})
+
+bot.startPolling();
+
+async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) {
+    Logging.debug("Before send", feed, items);
+    await Promise.all(feed.subscriber.map(
+        subscriber => Promise.all(
+            items.map(
+                item => bot.telegram.sendMessage(
+                    subscriber,
+                    item.guid + "\n" + entities.decode(item.title) + "\n\n" + entities.decode(item.contentSnippet)
+                ).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
+            )
+        )
+    ));
+}
+
+function checkAll() {
+    Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)));
+}
+
+setInterval(() => {
+    checkAll();
+}, 1000 * 60 * 60);
+checkAll();
+
+// import { AllHtmlEntities } from "html-entities";
+// import { fstat } from "fs";
+// const entities = new AllHtmlEntities();
+// async function sendFeedMail(feed: Feed, item: IFeedItem) {
+//     Logging.log("Sending Mail!")
+//     let mailOptions = {
+//         from: process.env.MAIL_SENDER,
+//         to: process.env.MAIL_RECEIVER,
+//         subject: '[OSPLUS_UPDATE] ' + entities.decode(item.title),
+//         text: entities.decode(item.contentSnippet),
+//         html: item.content
+//     };
+
+//     let info = await transporter.sendMail(mailOptions);
+//     Logging.log('Message sent: %s', info.messageId);
+//     db.set(calculateHash(item), true).write();
+// }

tsconfig.json

@@ -0,0 +1,14 @@
+{
+    "compilerOptions": {
+        "target": "es2017",
+        "module": "commonjs",
+        "moduleResolution": "node",
+        "outDir": "./lib",
+        "strict": false,
+        "esModuleInterop": true,
+        "preserveWatchOutput": true
+    },
+    "include": [
+        "./src"
+    ]
+}