This commit is contained in:
parent
422ebd0703
commit
9dfb1342e5
3
.gitignore
vendored
3
.gitignore
vendored
@ -4,4 +4,5 @@ logs/
|
||||
db.json
|
||||
lib/
|
||||
.yarn/cache
|
||||
.yarn/install-state.gz
|
||||
.yarn/install-state.gz
|
||||
persist/
|
26
Dockerfile
26
Dockerfile
@ -1,21 +1,29 @@
|
||||
FROM node:19
|
||||
FROM node:19 as builder
|
||||
|
||||
RUN mkdir -p /app
|
||||
WORKDIR /app
|
||||
|
||||
COPY ["package.json", "yarn.lock", ".yarnrc.yml", "/app/"]
|
||||
COPY ["src", "/app/src"]
|
||||
|
||||
RUN yarn install
|
||||
RUN yarn build
|
||||
|
||||
FROM node:19
|
||||
|
||||
LABEL maintainer="Fabian Stamm <dev@fabianstamm.de>"
|
||||
|
||||
RUN apt-get update
|
||||
|
||||
RUN mkdir -p /usr/src/app
|
||||
WORKDIR /usr/src/app
|
||||
RUN mkdir -p /app
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV=production
|
||||
|
||||
COPY ["package.json", "yarn.lock", ".yarnrc.yml", "/usr/src/app/"]
|
||||
COPY ".yarn" /usr/src/app/.yarn
|
||||
|
||||
RUN yarn install
|
||||
|
||||
COPY lib/ /usr/src/app/lib
|
||||
COPY --from=builder lib/ /app/lib
|
||||
|
||||
VOLUME [ "/usr/src/app/logs", "/usr/src/app/persist"]
|
||||
VOLUME [ "/app/logs", "/app/persist"]
|
||||
|
||||
CMD ["npm", "run", "start"]
|
||||
CMD ["node", "lib/index.js"]
|
2
example_feeds.txt
Normal file
2
example_feeds.txt
Normal file
@ -0,0 +1,2 @@
|
||||
https://www.netcup-sonderangebote.de/feed
|
||||
https://xkcd.com/rss.xml
|
15
package.json
15
package.json
@ -9,7 +9,9 @@
|
||||
"scripts": {
|
||||
"start": "node lib/index.js",
|
||||
"build": "tsc",
|
||||
"dev": "nodemon -e ts --exec 'node --loader ts-node/esm src/index.ts'"
|
||||
"dev-ts": "tsc -w",
|
||||
"dev-node": "nodemon lib/index.js",
|
||||
"dev": "run-p dev-ts dev-node"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/dotenv": "^8.2.0",
|
||||
@ -17,6 +19,7 @@
|
||||
"@types/node": "^18.15.11",
|
||||
"@types/node-fetch": "^3.0.3",
|
||||
"nodemon": "^2.0.22",
|
||||
"npm-run-all": "^4.1.5",
|
||||
"ts-node": "^10.9.1",
|
||||
"typescript": "^5.0.4"
|
||||
},
|
||||
@ -28,9 +31,15 @@
|
||||
"lodash": "^4.17.21",
|
||||
"lowdb": "^5.1.0",
|
||||
"node-fetch": "^3.3.1",
|
||||
"pino": "^8.11.0",
|
||||
"pino-pretty": "^10.0.0",
|
||||
"reflect-metadata": "^0.1.13",
|
||||
"rss-parser": "^3.13.0",
|
||||
"run-p": "^0.0.0",
|
||||
"sqlite3": "^5.1.6",
|
||||
"ssl-root-cas": "^1.3.1",
|
||||
"telegraf": "^4.12.2"
|
||||
"telegraf": "^4.12.2",
|
||||
"typeorm": "^0.3.15"
|
||||
},
|
||||
"packageManager": "yarn@3.5.0"
|
||||
}
|
||||
}
|
||||
|
25
src/data_source.ts
Normal file
25
src/data_source.ts
Normal file
@ -0,0 +1,25 @@
|
||||
import "reflect-metadata";
|
||||
import { DataSource } from "typeorm";
|
||||
import { Feed } from "./models/Feed.js";
|
||||
import { User } from "./models/User.js";
|
||||
import { Post } from "./models/Post.js";
|
||||
import Logging from "./log.js";
|
||||
|
||||
export const AppDataSource = new DataSource({
|
||||
type: "sqlite",
|
||||
database: "persist/db.sqlite",
|
||||
logging: false,
|
||||
entities: [User, Feed, Post],
|
||||
subscribers: [],
|
||||
migrations: [],
|
||||
})
|
||||
|
||||
export const appDataSourceReady = AppDataSource.initialize().then(() => {
|
||||
Logging.info("Database initialized!");
|
||||
return AppDataSource.synchronize().then(() => {
|
||||
Logging.info("Database synchronized!");
|
||||
})
|
||||
}).catch((err) => {
|
||||
Logging.info("Database initialization failed!");
|
||||
Logging.error(err);
|
||||
});
|
117
src/feed_fetcher.ts
Normal file
117
src/feed_fetcher.ts
Normal file
@ -0,0 +1,117 @@
|
||||
import { LessThan } from "typeorm";
|
||||
import { AppDataSource } from "./data_source.js";
|
||||
import Logging from "./log.js";
|
||||
import { Feed } from "./models/Feed.js";
|
||||
import { createHash } from "crypto";
|
||||
import rss from "rss-parser";
|
||||
import { Post } from "./models/Post.js";
|
||||
import bot from "./telegram.js";
|
||||
|
||||
interface RSSFeed {
|
||||
title: string;
|
||||
language: string;
|
||||
description: string;
|
||||
copyright: string;
|
||||
|
||||
items: RSSFeedItem[];
|
||||
}
|
||||
|
||||
interface RSSFeedItem {
|
||||
title: string;
|
||||
link: string;
|
||||
content: string,
|
||||
contentSnippet: string;
|
||||
guid: string;
|
||||
}
|
||||
|
||||
function calculateHash(item: RSSFeedItem) {
|
||||
let hash = createHash("sha512");
|
||||
if (item.content)
|
||||
hash.update(item.content);
|
||||
if (item.guid)
|
||||
hash.update(item.guid);
|
||||
if (item.title)
|
||||
hash.update(item.title);
|
||||
if (item.link)
|
||||
hash.update(item.link);
|
||||
if (item.contentSnippet)
|
||||
hash.update(item.contentSnippet);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function checkFeed(feed: Feed) {
|
||||
Logging.info("Fetching feed: %s", feed.url);
|
||||
let data = await fetch(feed.url).then(res => res.text());
|
||||
Logging.info("Received Data");
|
||||
Logging.debug(data);
|
||||
|
||||
const parser = new rss();
|
||||
Logging.info("Parsing feed");
|
||||
let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as RSSFeed;
|
||||
Logging.info("Parsed feed");
|
||||
|
||||
Logging.info("Checking for new items");
|
||||
|
||||
|
||||
for (let entry of feedData.items) {
|
||||
let hash = calculateHash(entry);
|
||||
let post = await AppDataSource.manager.findOne(Post, {
|
||||
where: {
|
||||
feed,
|
||||
hash
|
||||
}
|
||||
});
|
||||
if (!post) {
|
||||
post = AppDataSource.manager.create(Post, {
|
||||
feed,
|
||||
hash
|
||||
})
|
||||
|
||||
const decode = (inp) => inp;
|
||||
|
||||
feed.subscriber.map(async sub => {
|
||||
bot.telegram.sendMessage(
|
||||
sub.chatid,
|
||||
entry.guid + "\n" + decode(entry.title) + "\n\n" + decode(entry.contentSnippet)
|
||||
).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
|
||||
})
|
||||
|
||||
await AppDataSource.manager.save(post);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
|
||||
// const newItems = feedData.items.filter(item => !feed.itemExists(item));
|
||||
|
||||
// // Sending notifications
|
||||
// await sendFeedTelegraf(feed, newItems);
|
||||
|
||||
// feed.addItems(newItems);
|
||||
}
|
||||
|
||||
// const FEED_CHECK_INTERVAL = 1000 * 60 * 60;
|
||||
const FEED_CHECK_INTERVAL = 1000 * 30;
|
||||
|
||||
export default async function checkFeeds() {
|
||||
while (true) {
|
||||
let feed = await AppDataSource.manager.findOne(Feed, {
|
||||
where: {
|
||||
lastCheck: LessThan(new Date(Date.now() - FEED_CHECK_INTERVAL))
|
||||
},
|
||||
relations: {
|
||||
subscriber: true
|
||||
}
|
||||
})
|
||||
|
||||
if (!feed) {
|
||||
await new Promise(y => setTimeout(y, 1000));
|
||||
continue;
|
||||
}
|
||||
|
||||
await checkFeed(feed);
|
||||
|
||||
feed.lastCheck = new Date();
|
||||
await AppDataSource.manager.save(feed);
|
||||
}
|
||||
}
|
244
src/index.ts
244
src/index.ts
@ -1,230 +1,18 @@
|
||||
// require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create();
|
||||
// require('ssl-root-cas').inject();
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
||||
|
||||
import * as _Logging from "@hibas123/nodelogging";
|
||||
import { createHash } from "crypto";
|
||||
import * as dotenv from "dotenv";
|
||||
import { decode } from "html-entities";
|
||||
import { AppDataSource, appDataSourceReady } from "./data_source.js";
|
||||
import checkFeeds from "./feed_fetcher.js";
|
||||
import Logging from "./log.js";
|
||||
import { User } from "./models/User.js";
|
||||
import bot from "./telegram.js";
|
||||
|
||||
import fetch from "node-fetch";
|
||||
import rss from "rss-parser";
|
||||
import { Telegraf } from "telegraf";
|
||||
import { message } from "telegraf/filters";
|
||||
import * as fs from "fs";
|
||||
import lodash from 'lodash'
|
||||
|
||||
const Logging = (_Logging.default as any).default as typeof _Logging.default;
|
||||
|
||||
Promise.resolve().then(async () => {
|
||||
const { Low, LowSync } = await import("lowdb");
|
||||
// @ts-ignore
|
||||
const { JSONFileSync } = await import("lowdb/node") as any;
|
||||
|
||||
class LowWithLodash<T> extends LowSync<T> {
|
||||
chain: lodash.ExpChain<this['data']> = lodash.chain(this).get('data')
|
||||
}
|
||||
|
||||
dotenv.config();
|
||||
const parser = new rss();
|
||||
// const entities = new AllHtmlEntities();
|
||||
|
||||
if (!fs.existsSync("./persist")) {
|
||||
fs.mkdirSync("./persist")
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
const adapter = new JSONFileSync<{ feeds: IDBFeed[] }>("persist/db.json");
|
||||
const db: any = new LowWithLodash(adapter);
|
||||
db.read();
|
||||
|
||||
db.chain.defaults({ feeds: [] });
|
||||
db.write();
|
||||
|
||||
interface IDBFeed {
|
||||
url: string;
|
||||
subscriber: number[];
|
||||
oldEntries: string[];
|
||||
}
|
||||
|
||||
class Database {
|
||||
static findFeed(url: string): DBFeed {
|
||||
const feed = db.chain.get("feeds").find(e => e.url === url).value();
|
||||
return feed ? new DBFeed(feed) : undefined;
|
||||
}
|
||||
|
||||
|
||||
static addFeed(url: string): IDBFeed {
|
||||
const feed = {
|
||||
url: url,
|
||||
oldEntries: [],
|
||||
subscriber: []
|
||||
};
|
||||
db.chain.get("feeds").unshift(feed)
|
||||
db.write();
|
||||
return feed;
|
||||
}
|
||||
|
||||
static addSubscriber(url: string, chatid: number) {
|
||||
const feed = this.findFeed(url);
|
||||
if (!feed)
|
||||
this.addFeed(url);
|
||||
else {
|
||||
if (feed.subscriber.some(e => e === chatid)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
db.chain.get("feeds").find(e => e.url === url).get("subscriber").push(chatid)
|
||||
db.write();
|
||||
}
|
||||
|
||||
static findSubscribed(chatid: number) {
|
||||
return db.chain.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value();
|
||||
}
|
||||
|
||||
static removeSubscriber(url: string, chatid: number) {
|
||||
db.chain.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid)
|
||||
db.write();
|
||||
}
|
||||
|
||||
static addItems(url: string, hashes: string[]) {
|
||||
db.chain.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes)
|
||||
db.write();
|
||||
}
|
||||
|
||||
static getAll() {
|
||||
return db.chain.get("feeds").map(feed => new DBFeed(feed)).value();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class DBFeed implements IDBFeed {
|
||||
url: string;
|
||||
subscriber: number[];
|
||||
oldEntries: string[];
|
||||
|
||||
constructor(dbobject?: IDBFeed) {
|
||||
if (dbobject) {
|
||||
for (let key in dbobject)
|
||||
this[key] = dbobject[key];
|
||||
}
|
||||
}
|
||||
|
||||
itemExists(item: IFeedItem): boolean {
|
||||
const hash = calculateHash(item);
|
||||
return !!this.oldEntries.find(e => e === hash);
|
||||
}
|
||||
|
||||
addItems(items: IFeedItem[]) {
|
||||
Database.addItems(this.url, items.map(item => calculateHash(item)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
interface Feed {
|
||||
title: string;
|
||||
language: string;
|
||||
description: string;
|
||||
copyright: string;
|
||||
|
||||
items: IFeedItem[];
|
||||
}
|
||||
|
||||
interface IFeedItem {
|
||||
title: string;
|
||||
link: string;
|
||||
content: string,
|
||||
contentSnippet: string;
|
||||
guid: string;
|
||||
}
|
||||
|
||||
function calculateHash(item: IFeedItem) {
|
||||
let hash = createHash("sha512");
|
||||
if (item.content)
|
||||
hash.update(item.content);
|
||||
if (item.guid)
|
||||
hash.update(item.guid);
|
||||
if (item.title)
|
||||
hash.update(item.title);
|
||||
if (item.link)
|
||||
hash.update(item.link);
|
||||
if (item.contentSnippet)
|
||||
hash.update(item.contentSnippet);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function checkFeed(feed: DBFeed) {
|
||||
Logging.log("Fetching:", feed.url);
|
||||
let data = await fetch(feed.url).then(res => res.text());
|
||||
Logging.log("Received Data");
|
||||
|
||||
let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed;
|
||||
|
||||
// Check for new items
|
||||
Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
|
||||
const newItems = feedData.items.filter(item => !feed.itemExists(item));
|
||||
|
||||
// Sending notifications
|
||||
await sendFeedTelegraf(feed, newItems);
|
||||
|
||||
feed.addItems(newItems);
|
||||
}
|
||||
|
||||
|
||||
const bot = new Telegraf(process.env.TG_TOKEN)
|
||||
|
||||
bot.start(async ctx => {
|
||||
await ctx.reply("Send some RSS Feed URLs to get started.");
|
||||
})
|
||||
|
||||
bot.command("list", async (ctx) => {
|
||||
const chatid = ctx.chat.id;
|
||||
const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n");
|
||||
await ctx.reply("You are currently subscribed to: \n" + feeds);
|
||||
})
|
||||
|
||||
bot.on(message("text", "entities"), async ctx => {
|
||||
const chatid = ctx.chat.id;
|
||||
Logging.debug("Message From:", chatid, ctx.message);
|
||||
|
||||
const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length));
|
||||
await Promise.all(urls.map(async url => {
|
||||
Database.addSubscriber(url, chatid);
|
||||
await ctx.reply("Subscribed to: " + url);
|
||||
let feed = Database.findFeed(url)
|
||||
if (feed)
|
||||
checkFeed(feed);
|
||||
else
|
||||
Logging.error("Cannot find created feed!")
|
||||
}))
|
||||
if (urls.length === 0)
|
||||
await ctx.reply("No URLs found in message.");
|
||||
})
|
||||
|
||||
bot.launch()
|
||||
|
||||
async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) {
|
||||
Logging.debug("Before send", feed, items);
|
||||
await Promise.all(feed.subscriber.map(
|
||||
subscriber => Promise.all(
|
||||
items.map(
|
||||
item => bot.telegram.sendMessage(
|
||||
subscriber,
|
||||
item.guid + "\n" + decode(item.title) + "\n\n" + decode(item.contentSnippet)
|
||||
).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
|
||||
)
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
function checkAll() {
|
||||
Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)).finally(() => {
|
||||
db.write();
|
||||
}));
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
checkAll();
|
||||
}, 1000 * 60 * 60);
|
||||
checkAll();
|
||||
})
|
||||
appDataSourceReady.then(() => {
|
||||
return Promise.allSettled([
|
||||
bot.launch({}),
|
||||
checkFeeds()
|
||||
])
|
||||
}).then(() => {
|
||||
Logging.info("Exiting...");
|
||||
}).catch(err => {
|
||||
Logging.error(err);
|
||||
process.exit(1);
|
||||
});
|
12
src/log.ts
Normal file
12
src/log.ts
Normal file
@ -0,0 +1,12 @@
|
||||
import pino from "pino";
|
||||
|
||||
const Logging = pino({
|
||||
transport: {
|
||||
target: "pino-pretty",
|
||||
options: {
|
||||
colorize: true
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
export default Logging;
|
0
src/migrate.ts
Normal file
0
src/migrate.ts
Normal file
24
src/models/Feed.ts
Normal file
24
src/models/Feed.ts
Normal file
@ -0,0 +1,24 @@
|
||||
import { Entity, Unique, PrimaryGeneratedColumn, Column, ManyToMany, JoinTable, OneToMany, Relation } from "typeorm";
|
||||
import { User } from "./User.js";
|
||||
import { Post } from "./Post.js";
|
||||
|
||||
@Entity()
|
||||
export class Feed {
|
||||
@PrimaryGeneratedColumn()
|
||||
id: number;
|
||||
|
||||
@Column()
|
||||
url: string;
|
||||
|
||||
@Column({
|
||||
default: "1970-01-01 01:00:00.000"
|
||||
})
|
||||
lastCheck?: Date;
|
||||
|
||||
@ManyToMany(() => User, user => user.feeds)
|
||||
@JoinTable()
|
||||
subscriber: Relation<User>[];
|
||||
|
||||
@OneToMany(() => Post, post => post.feed)
|
||||
oldEntries: Relation<Post>[];
|
||||
}
|
11
src/models/Post.ts
Normal file
11
src/models/Post.ts
Normal file
@ -0,0 +1,11 @@
|
||||
import { PrimaryColumn, Entity, ManyToOne, Relation } from "typeorm";
|
||||
import { Feed } from "./Feed.js";
|
||||
|
||||
@Entity()
|
||||
export class Post {
|
||||
@PrimaryColumn()
|
||||
hash: string;
|
||||
|
||||
@ManyToOne(() => Feed, (feed) => feed.oldEntries)
|
||||
feed: Relation<Feed>;
|
||||
}
|
15
src/models/User.ts
Normal file
15
src/models/User.ts
Normal file
@ -0,0 +1,15 @@
|
||||
import { PrimaryGeneratedColumn, ManyToMany, Relation, Entity, Column, Unique } from "typeorm"
|
||||
import { Feed } from "./Feed.js";
|
||||
|
||||
@Entity()
|
||||
@Unique("chatid", ["chatid"])
|
||||
export class User {
|
||||
@PrimaryGeneratedColumn()
|
||||
id: number;
|
||||
|
||||
@Column()
|
||||
chatid: number;
|
||||
|
||||
@ManyToMany(() => Feed, feed => feed.subscriber)
|
||||
feeds: Relation<Feed>[];
|
||||
}
|
230
src/old_index.ts
Normal file
230
src/old_index.ts
Normal file
@ -0,0 +1,230 @@
|
||||
// require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create();
|
||||
// require('ssl-root-cas').inject();
|
||||
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
|
||||
|
||||
import * as _Logging from "@hibas123/nodelogging";
|
||||
import { createHash } from "crypto";
|
||||
import * as dotenv from "dotenv";
|
||||
import { decode } from "html-entities";
|
||||
|
||||
import fetch from "node-fetch";
|
||||
import rss from "rss-parser";
|
||||
import { Telegraf } from "telegraf";
|
||||
import { message } from "telegraf/filters";
|
||||
import * as fs from "fs";
|
||||
import lodash from 'lodash'
|
||||
|
||||
const Logging = (_Logging.default as any).default as typeof _Logging.default;
|
||||
|
||||
Promise.resolve().then(async () => {
|
||||
const { Low, LowSync } = await import("lowdb");
|
||||
// @ts-ignore
|
||||
const { JSONFileSync } = await import("lowdb/node") as any;
|
||||
|
||||
class LowWithLodash<T> extends LowSync<T> {
|
||||
chain: lodash.ExpChain<this['data']> = lodash.chain(this).get('data')
|
||||
}
|
||||
|
||||
dotenv.config();
|
||||
const parser = new rss();
|
||||
// const entities = new AllHtmlEntities();
|
||||
|
||||
if (!fs.existsSync("./persist")) {
|
||||
fs.mkdirSync("./persist")
|
||||
}
|
||||
|
||||
// @ts-ignore
|
||||
const adapter = new JSONFileSync<{ feeds: IDBFeed[] }>("persist/db.json");
|
||||
const db: any = new LowWithLodash(adapter);
|
||||
db.read();
|
||||
|
||||
db.chain.defaults({ feeds: [] });
|
||||
db.write();
|
||||
|
||||
interface IDBFeed {
|
||||
url: string;
|
||||
subscriber: number[];
|
||||
oldEntries: string[];
|
||||
}
|
||||
|
||||
class Database {
|
||||
static findFeed(url: string): DBFeed {
|
||||
const feed = db.chain.get("feeds").find(e => e.url === url).value();
|
||||
return feed ? new DBFeed(feed) : undefined;
|
||||
}
|
||||
|
||||
|
||||
static addFeed(url: string): IDBFeed {
|
||||
const feed = {
|
||||
url: url,
|
||||
oldEntries: [],
|
||||
subscriber: []
|
||||
};
|
||||
db.chain.get("feeds").unshift(feed)
|
||||
db.write();
|
||||
return feed;
|
||||
}
|
||||
|
||||
static addSubscriber(url: string, chatid: number) {
|
||||
const feed = this.findFeed(url);
|
||||
if (!feed)
|
||||
this.addFeed(url);
|
||||
else {
|
||||
if (feed.subscriber.some(e => e === chatid)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
db.chain.get("feeds").find(e => e.url === url).get("subscriber").push(chatid)
|
||||
db.write();
|
||||
}
|
||||
|
||||
static findSubscribed(chatid: number) {
|
||||
return db.chain.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value();
|
||||
}
|
||||
|
||||
static removeSubscriber(url: string, chatid: number) {
|
||||
db.chain.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid)
|
||||
db.write();
|
||||
}
|
||||
|
||||
static addItems(url: string, hashes: string[]) {
|
||||
db.chain.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes)
|
||||
db.write();
|
||||
}
|
||||
|
||||
static getAll() {
|
||||
return db.chain.get("feeds").map(feed => new DBFeed(feed)).value();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class DBFeed implements IDBFeed {
|
||||
url: string;
|
||||
subscriber: number[];
|
||||
oldEntries: string[];
|
||||
|
||||
constructor(dbobject?: IDBFeed) {
|
||||
if (dbobject) {
|
||||
for (let key in dbobject)
|
||||
this[key] = dbobject[key];
|
||||
}
|
||||
}
|
||||
|
||||
itemExists(item: IFeedItem): boolean {
|
||||
const hash = calculateHash(item);
|
||||
return !!this.oldEntries.find(e => e === hash);
|
||||
}
|
||||
|
||||
addItems(items: IFeedItem[]) {
|
||||
Database.addItems(this.url, items.map(item => calculateHash(item)));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
interface Feed {
|
||||
title: string;
|
||||
language: string;
|
||||
description: string;
|
||||
copyright: string;
|
||||
|
||||
items: IFeedItem[];
|
||||
}
|
||||
|
||||
interface IFeedItem {
|
||||
title: string;
|
||||
link: string;
|
||||
content: string,
|
||||
contentSnippet: string;
|
||||
guid: string;
|
||||
}
|
||||
|
||||
function calculateHash(item: IFeedItem) {
|
||||
let hash = createHash("sha512");
|
||||
if (item.content)
|
||||
hash.update(item.content);
|
||||
if (item.guid)
|
||||
hash.update(item.guid);
|
||||
if (item.title)
|
||||
hash.update(item.title);
|
||||
if (item.link)
|
||||
hash.update(item.link);
|
||||
if (item.contentSnippet)
|
||||
hash.update(item.contentSnippet);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function checkFeed(feed: DBFeed) {
|
||||
Logging.log("Fetching:", feed.url);
|
||||
let data = await fetch(feed.url).then(res => res.text());
|
||||
Logging.log("Received Data");
|
||||
|
||||
let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed;
|
||||
|
||||
// Check for new items
|
||||
Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
|
||||
const newItems = feedData.items.filter(item => !feed.itemExists(item));
|
||||
|
||||
// Sending notifications
|
||||
await sendFeedTelegraf(feed, newItems);
|
||||
|
||||
feed.addItems(newItems);
|
||||
}
|
||||
|
||||
|
||||
const bot = new Telegraf(process.env.TG_TOKEN)
|
||||
|
||||
bot.start(async ctx => {
|
||||
await ctx.reply("Send some RSS Feed URLs to get started.");
|
||||
})
|
||||
|
||||
bot.command("list", async (ctx) => {
|
||||
const chatid = ctx.chat.id;
|
||||
const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n");
|
||||
await ctx.reply("You are currently subscribed to: \n" + feeds);
|
||||
})
|
||||
|
||||
bot.on(message("text", "entities"), async ctx => {
|
||||
const chatid = ctx.chat.id;
|
||||
Logging.debug("Message From:", chatid, ctx.message);
|
||||
|
||||
const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length));
|
||||
await Promise.all(urls.map(async url => {
|
||||
Database.addSubscriber(url, chatid);
|
||||
await ctx.reply("Subscribed to: " + url);
|
||||
let feed = Database.findFeed(url)
|
||||
if (feed)
|
||||
checkFeed(feed);
|
||||
else
|
||||
Logging.error("Cannot find created feed!")
|
||||
}))
|
||||
if (urls.length === 0)
|
||||
await ctx.reply("No URLs found in message.");
|
||||
})
|
||||
|
||||
bot.launch()
|
||||
|
||||
async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) {
|
||||
Logging.debug("Before send", feed, items);
|
||||
await Promise.all(feed.subscriber.map(
|
||||
subscriber => Promise.all(
|
||||
items.map(
|
||||
item => bot.telegram.sendMessage(
|
||||
subscriber,
|
||||
item.guid + "\n" + decode(item.title) + "\n\n" + decode(item.contentSnippet)
|
||||
).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
|
||||
)
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
function checkAll() {
|
||||
Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)).finally(() => {
|
||||
db.write();
|
||||
}));
|
||||
}
|
||||
|
||||
setInterval(() => {
|
||||
checkAll();
|
||||
}, 1000 * 60 * 60);
|
||||
checkAll();
|
||||
})
|
162
src/telegram.ts
Normal file
162
src/telegram.ts
Normal file
@ -0,0 +1,162 @@
|
||||
import { Markup, Telegraf } from "telegraf";
|
||||
import { message } from "telegraf/filters";
|
||||
import { AppDataSource } from "./data_source.js";
|
||||
import { User } from "./models/User.js";
|
||||
import Logging from "./log.js";
|
||||
import { Feed } from "./models/Feed.js";
|
||||
|
||||
|
||||
const bot = new Telegraf(process.env.TG_TOKEN)
|
||||
|
||||
function botHandler<T>(handler: (ctx: T) => Promise<any>) {
|
||||
return async (ctx: T) => {
|
||||
try {
|
||||
await handler(ctx);
|
||||
} catch (err) {
|
||||
Logging.error(err);
|
||||
(ctx as any).reply("An error occured while processing your request.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bot.start(botHandler(async ctx => {
|
||||
await ctx.reply("Send some RSS Feed URLs to get started.");
|
||||
}));
|
||||
|
||||
bot.command("list", botHandler(async (ctx) => {
|
||||
|
||||
let user = await AppDataSource.manager.findOne(User, {
|
||||
where: {
|
||||
chatid: ctx.chat.id
|
||||
},
|
||||
relations: {
|
||||
feeds: true,
|
||||
}
|
||||
})
|
||||
|
||||
if (!user) {
|
||||
ctx.reply("You have no feeds yet.");
|
||||
} else {
|
||||
ctx.reply("You are currently subscribed to: \n" + user.feeds.map(feed => feed.url).join("\n"));
|
||||
}
|
||||
}));
|
||||
|
||||
bot.command("delete", botHandler(async ctx => {
|
||||
let user = await AppDataSource.manager.findOne(User, {
|
||||
where: {
|
||||
chatid: ctx.chat.id
|
||||
},
|
||||
relations: {
|
||||
feeds: true,
|
||||
}
|
||||
})
|
||||
|
||||
if (!user || user.feeds.length === 0) {
|
||||
ctx.reply("You have no feeds yet.");
|
||||
} else {
|
||||
|
||||
ctx.replyWithHTML("Select a feed to delete:", {
|
||||
...Markup.inlineKeyboard(user.feeds.map(feed => Markup.button.callback(feed.url, "delete:" + feed.id)))
|
||||
})
|
||||
}
|
||||
}));
|
||||
|
||||
bot.on("callback_query", botHandler(async ctx => {
|
||||
Logging.info("Callback Query:");
|
||||
Logging.info(ctx.callbackQuery);
|
||||
|
||||
const remove_options = () => {
|
||||
ctx.editMessageReplyMarkup({
|
||||
inline_keyboard: []
|
||||
})
|
||||
}
|
||||
|
||||
let data = (ctx.callbackQuery as any).data;
|
||||
if (!data) {
|
||||
ctx.answerCbQuery("Invalid data");
|
||||
remove_options();
|
||||
return
|
||||
}
|
||||
|
||||
let [action, id] = data.split(":");
|
||||
if (!action || !id) {
|
||||
ctx.answerCbQuery("Invalid data");
|
||||
remove_options();
|
||||
return
|
||||
}
|
||||
|
||||
if (action === "delete") {
|
||||
let user = await AppDataSource.manager.findOne(User, {
|
||||
where: {
|
||||
chatid: ctx.chat.id
|
||||
},
|
||||
relations: {
|
||||
feeds: true,
|
||||
}
|
||||
});
|
||||
|
||||
if (user) {
|
||||
user.feeds = user.feeds.filter(feed => feed.id !== Number(id));
|
||||
await AppDataSource.manager.save(user);
|
||||
}
|
||||
} else {
|
||||
ctx.answerCbQuery("Invalid action");
|
||||
remove_options();
|
||||
return
|
||||
}
|
||||
|
||||
ctx.answerCbQuery("Success");
|
||||
remove_options();
|
||||
}));
|
||||
|
||||
bot.on(message("text", "entities"), botHandler(async ctx => {
|
||||
let user = await AppDataSource.manager.findOne(User, {
|
||||
where: {
|
||||
chatid: ctx.chat.id
|
||||
}
|
||||
});
|
||||
|
||||
if (!user) {
|
||||
user = AppDataSource.manager.create(User, {
|
||||
chatid: ctx.chat.id,
|
||||
feeds: [],
|
||||
})
|
||||
|
||||
await AppDataSource.manager.save(user);
|
||||
}
|
||||
|
||||
|
||||
Logging.debug("Received message:", ctx.message);
|
||||
const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.slice(e.offset, e.offset + e.length));
|
||||
Logging.debug("URLs:", urls);
|
||||
|
||||
for (let url of urls) {
|
||||
let normalized = new URL(url).href;
|
||||
Logging.debug("Normalized URL:", normalized);
|
||||
|
||||
let feed = await AppDataSource.manager.findOne(Feed, {
|
||||
where: {
|
||||
url: normalized
|
||||
},
|
||||
relations: {
|
||||
subscriber: true,
|
||||
}
|
||||
});
|
||||
|
||||
if (feed) {
|
||||
feed.subscriber.push(user);
|
||||
await AppDataSource.manager.save(feed);
|
||||
} else {
|
||||
feed = AppDataSource.manager.create(Feed, {
|
||||
oldEntries: [],
|
||||
subscriber: [user],
|
||||
url: normalized,
|
||||
})
|
||||
await AppDataSource.manager.save(feed);
|
||||
}
|
||||
}
|
||||
|
||||
await ctx.reply("Added feeds.");
|
||||
}));
|
||||
|
||||
export default bot;
|
@ -8,7 +8,9 @@
|
||||
"esModuleInterop": true,
|
||||
"preserveWatchOutput": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"skipLibCheck": true
|
||||
"skipLibCheck": true,
|
||||
"experimentalDecorators": true,
|
||||
"emitDecoratorMetadata": true
|
||||
},
|
||||
"include": ["./src"],
|
||||
"exclude": ["node_modules"]
|
||||
|
Loading…
Reference in New Issue
Block a user