// require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create(); // require('ssl-root-cas').inject(); process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0"; import * as _Logging from "@hibas123/nodelogging"; import { createHash } from "crypto"; import * as dotenv from "dotenv"; import { decode } from "html-entities"; import fetch from "node-fetch"; import rss from "rss-parser"; import { Telegraf } from "telegraf"; import { message } from "telegraf/filters"; import * as fs from "fs"; import lodash from 'lodash' const Logging = (_Logging.default as any).default as typeof _Logging.default; Promise.resolve().then(async () => { const { Low, LowSync } = await import("lowdb"); // @ts-ignore const { JSONFileSync } = await import("lowdb/node") as any; class LowWithLodash extends LowSync { chain: lodash.ExpChain = lodash.chain(this).get('data') } dotenv.config(); const parser = new rss(); // const entities = new AllHtmlEntities(); if (!fs.existsSync("./persist")) { fs.mkdirSync("./persist") } // @ts-ignore const adapter = new JSONFileSync<{ feeds: IDBFeed[] }>("persist/db.json"); const db: any = new LowWithLodash(adapter); db.read(); db.chain.defaults({ feeds: [] }); db.write(); interface IDBFeed { url: string; subscriber: number[]; oldEntries: string[]; } class Database { static findFeed(url: string): DBFeed { const feed = db.chain.get("feeds").find(e => e.url === url).value(); return feed ? new DBFeed(feed) : undefined; } static addFeed(url: string): IDBFeed { const feed = { url: url, oldEntries: [], subscriber: [] }; db.chain.get("feeds").unshift(feed) db.write(); return feed; } static addSubscriber(url: string, chatid: number) { const feed = this.findFeed(url); if (!feed) this.addFeed(url); else { if (feed.subscriber.some(e => e === chatid)) { return; } } db.chain.get("feeds").find(e => e.url === url).get("subscriber").push(chatid) db.write(); } static findSubscribed(chatid: number) { return db.chain.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value(); } static removeSubscriber(url: string, chatid: number) { db.chain.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid) db.write(); } static addItems(url: string, hashes: string[]) { db.chain.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes) db.write(); } static getAll() { return db.chain.get("feeds").map(feed => new DBFeed(feed)).value(); } } class DBFeed implements IDBFeed { url: string; subscriber: number[]; oldEntries: string[]; constructor(dbobject?: IDBFeed) { if (dbobject) { for (let key in dbobject) this[key] = dbobject[key]; } } itemExists(item: IFeedItem): boolean { const hash = calculateHash(item); return !!this.oldEntries.find(e => e === hash); } addItems(items: IFeedItem[]) { Database.addItems(this.url, items.map(item => calculateHash(item))); } } interface Feed { title: string; language: string; description: string; copyright: string; items: IFeedItem[]; } interface IFeedItem { title: string; link: string; content: string, contentSnippet: string; guid: string; } function calculateHash(item: IFeedItem) { let hash = createHash("sha512"); if (item.content) hash.update(item.content); if (item.guid) hash.update(item.guid); if (item.title) hash.update(item.title); if (item.link) hash.update(item.link); if (item.contentSnippet) hash.update(item.contentSnippet); return hash.digest("hex"); } async function checkFeed(feed: DBFeed) { Logging.log("Fetching:", feed.url); let data = await fetch(feed.url).then(res => res.text()); Logging.log("Received Data"); let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed; // Check for new items Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e))); const newItems = feedData.items.filter(item => !feed.itemExists(item)); // Sending notifications await sendFeedTelegraf(feed, newItems); feed.addItems(newItems); } const bot = new Telegraf(process.env.TG_TOKEN) bot.start(async ctx => { await ctx.reply("Send some RSS Feed URLs to get started."); }) bot.command("list", async (ctx) => { const chatid = ctx.chat.id; const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n"); await ctx.reply("You are currently subscribed to: \n" + feeds); }) bot.on(message("text", "entities"), async ctx => { const chatid = ctx.chat.id; Logging.debug("Message From:", chatid, ctx.message); const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length)); await Promise.all(urls.map(async url => { Database.addSubscriber(url, chatid); await ctx.reply("Subscribed to: " + url); let feed = Database.findFeed(url) if (feed) checkFeed(feed); else Logging.error("Cannot find created feed!") })) if (urls.length === 0) await ctx.reply("No URLs found in message."); }) bot.launch() async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) { Logging.debug("Before send", feed, items); await Promise.all(feed.subscriber.map( subscriber => Promise.all( items.map( item => bot.telegram.sendMessage( subscriber, item.guid + "\n" + decode(item.title) + "\n\n" + decode(item.contentSnippet) ).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent")) ) ) )); } function checkAll() { Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)).finally(() => { db.write(); })); } setInterval(() => { checkAll(); }, 1000 * 60 * 60); checkAll(); })