TelegramRSS/src/old_index.ts

// require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create();
// require('ssl-root-cas').inject();
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";

import * as _Logging from "@hibas123/nodelogging";
import { createHash } from "crypto";
import * as dotenv from "dotenv";
import { decode } from "html-entities";

import fetch from "node-fetch";
import rss from "rss-parser";
import { Telegraf } from "telegraf";
import { message } from "telegraf/filters";
import * as fs from "fs";
import lodash from 'lodash'

const Logging = (_Logging.default as any).default as typeof _Logging.default;

Promise.resolve().then(async () => {
    const { Low, LowSync } = await import("lowdb");
    // @ts-ignore
    const { JSONFileSync } = await import("lowdb/node") as any;

    class LowWithLodash<T> extends LowSync<T> {
        chain: lodash.ExpChain<this['data']> = lodash.chain(this).get('data')
    }

    dotenv.config();
    const parser = new rss();
    // const entities = new AllHtmlEntities();

    if (!fs.existsSync("./persist")) {
        fs.mkdirSync("./persist")
    }

    // @ts-ignore
    const adapter = new JSONFileSync<{ feeds: IDBFeed[] }>("persist/db.json");
    const db: any = new LowWithLodash(adapter);
    db.read();

    db.chain.defaults({ feeds: [] });
    db.write();

    interface IDBFeed {
        url: string;
        subscriber: number[];
        oldEntries: string[];
    }

    class Database {
        static findFeed(url: string): DBFeed {
            const feed = db.chain.get("feeds").find(e => e.url === url).value();
            return feed ? new DBFeed(feed) : undefined;
        }


        static addFeed(url: string): IDBFeed {
            const feed = {
                url: url,
                oldEntries: [],
                subscriber: []
            };
            db.chain.get("feeds").unshift(feed)
            db.write();
            return feed;
        }

        static addSubscriber(url: string, chatid: number) {
            const feed = this.findFeed(url);
            if (!feed)
                this.addFeed(url);
            else {
                if (feed.subscriber.some(e => e === chatid)) {
                    return;
                }
            }
            db.chain.get("feeds").find(e => e.url === url).get("subscriber").push(chatid)
            db.write();
        }

        static findSubscribed(chatid: number) {
            return db.chain.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value();
        }

        static removeSubscriber(url: string, chatid: number) {
            db.chain.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid)
            db.write();
        }

        static addItems(url: string, hashes: string[]) {
            db.chain.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes)
            db.write();
        }

        static getAll() {
            return db.chain.get("feeds").map(feed => new DBFeed(feed)).value();
        }
    }


    class DBFeed implements IDBFeed {
        url: string;
        subscriber: number[];
        oldEntries: string[];

        constructor(dbobject?: IDBFeed) {
            if (dbobject) {
                for (let key in dbobject)
                    this[key] = dbobject[key];
            }
        }

        itemExists(item: IFeedItem): boolean {
            const hash = calculateHash(item);
            return !!this.oldEntries.find(e => e === hash);
        }

        addItems(items: IFeedItem[]) {
            Database.addItems(this.url, items.map(item => calculateHash(item)));
        }
    }


    interface Feed {
        title: string;
        language: string;
        description: string;
        copyright: string;

        items: IFeedItem[];
    }

    interface IFeedItem {
        title: string;
        link: string;
        content: string,
        contentSnippet: string;
        guid: string;
    }

    function calculateHash(item: IFeedItem) {
        let hash = createHash("sha512");
        if (item.content)
            hash.update(item.content);
        if (item.guid)
            hash.update(item.guid);
        if (item.title)
            hash.update(item.title);
        if (item.link)
            hash.update(item.link);
        if (item.contentSnippet)
            hash.update(item.contentSnippet);
        return hash.digest("hex");
    }

    async function checkFeed(feed: DBFeed) {
        Logging.log("Fetching:", feed.url);
        let data = await fetch(feed.url).then(res => res.text());
        Logging.log("Received Data");

        let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed;

        // Check for new items
        Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
        const newItems = feedData.items.filter(item => !feed.itemExists(item));

        // Sending notifications
        await sendFeedTelegraf(feed, newItems);

        feed.addItems(newItems);
    }


    const bot = new Telegraf(process.env.TG_TOKEN)

    bot.start(async ctx => {
        await ctx.reply("Send some RSS Feed URLs to get started.");
    })

    bot.command("list", async (ctx) => {
        const chatid = ctx.chat.id;
        const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n");
        await ctx.reply("You are currently subscribed to: \n" + feeds);
    })

    bot.on(message("text", "entities"), async ctx => {
        const chatid = ctx.chat.id;
        Logging.debug("Message From:", chatid, ctx.message);

        const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length));
        await Promise.all(urls.map(async url => {
            Database.addSubscriber(url, chatid);
            await ctx.reply("Subscribed to: " + url);
            let feed = Database.findFeed(url)
            if (feed)
                checkFeed(feed);
            else
                Logging.error("Cannot find created feed!")
        }))
        if (urls.length === 0)
            await ctx.reply("No URLs found in message.");
    })

    bot.launch()

    async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) {
        Logging.debug("Before send", feed, items);
        await Promise.all(feed.subscriber.map(
            subscriber => Promise.all(
                items.map(
                    item => bot.telegram.sendMessage(
                        subscriber,
                        item.guid + "\n" + decode(item.title) + "\n\n" + decode(item.contentSnippet)
                    ).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
                )
            )
        ));
    }

    function checkAll() {
        Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)).finally(() => {
            db.write();
        }));
    }

    setInterval(() => {
        checkAll();
    }, 1000 * 60 * 60);
    checkAll();
})