New version of this thing
continuous-integration/drone/push Build is failing Details

This commit is contained in:
Fabian Stamm 2023-04-26 00:04:56 +02:00
parent 422ebd0703
commit 9dfb1342e5
16 changed files with 2462 additions and 265 deletions

3
.gitignore vendored
View File

@ -4,4 +4,5 @@ logs/
db.json
lib/
.yarn/cache
.yarn/install-state.gz
.yarn/install-state.gz
persist/

View File

@ -1,21 +1,29 @@
FROM node:19
FROM node:19 as builder
RUN mkdir -p /app
WORKDIR /app
COPY ["package.json", "yarn.lock", ".yarnrc.yml", "/app/"]
COPY ["src", "/app/src"]
RUN yarn install
RUN yarn build
FROM node:19
LABEL maintainer="Fabian Stamm <dev@fabianstamm.de>"
RUN apt-get update
RUN mkdir -p /usr/src/app
WORKDIR /usr/src/app
RUN mkdir -p /app
WORKDIR /app
ENV NODE_ENV=production
COPY ["package.json", "yarn.lock", ".yarnrc.yml", "/usr/src/app/"]
COPY ".yarn" /usr/src/app/.yarn
RUN yarn install
COPY lib/ /usr/src/app/lib
COPY --from=builder lib/ /app/lib
VOLUME [ "/usr/src/app/logs", "/usr/src/app/persist"]
VOLUME [ "/app/logs", "/app/persist"]
CMD ["npm", "run", "start"]
CMD ["node", "lib/index.js"]

2
example_feeds.txt Normal file
View File

@ -0,0 +1,2 @@
https://www.netcup-sonderangebote.de/feed
https://xkcd.com/rss.xml

View File

@ -9,7 +9,9 @@
"scripts": {
"start": "node lib/index.js",
"build": "tsc",
"dev": "nodemon -e ts --exec 'node --loader ts-node/esm src/index.ts'"
"dev-ts": "tsc -w",
"dev-node": "nodemon lib/index.js",
"dev": "run-p dev-ts dev-node"
},
"devDependencies": {
"@types/dotenv": "^8.2.0",
@ -17,6 +19,7 @@
"@types/node": "^18.15.11",
"@types/node-fetch": "^3.0.3",
"nodemon": "^2.0.22",
"npm-run-all": "^4.1.5",
"ts-node": "^10.9.1",
"typescript": "^5.0.4"
},
@ -28,9 +31,15 @@
"lodash": "^4.17.21",
"lowdb": "^5.1.0",
"node-fetch": "^3.3.1",
"pino": "^8.11.0",
"pino-pretty": "^10.0.0",
"reflect-metadata": "^0.1.13",
"rss-parser": "^3.13.0",
"run-p": "^0.0.0",
"sqlite3": "^5.1.6",
"ssl-root-cas": "^1.3.1",
"telegraf": "^4.12.2"
"telegraf": "^4.12.2",
"typeorm": "^0.3.15"
},
"packageManager": "yarn@3.5.0"
}
}

25
src/data_source.ts Normal file
View File

@ -0,0 +1,25 @@
import "reflect-metadata";
import { DataSource } from "typeorm";
import { Feed } from "./models/Feed.js";
import { User } from "./models/User.js";
import { Post } from "./models/Post.js";
import Logging from "./log.js";
export const AppDataSource = new DataSource({
type: "sqlite",
database: "persist/db.sqlite",
logging: false,
entities: [User, Feed, Post],
subscribers: [],
migrations: [],
})
export const appDataSourceReady = AppDataSource.initialize().then(() => {
Logging.info("Database initialized!");
return AppDataSource.synchronize().then(() => {
Logging.info("Database synchronized!");
})
}).catch((err) => {
Logging.info("Database initialization failed!");
Logging.error(err);
});

117
src/feed_fetcher.ts Normal file
View File

@ -0,0 +1,117 @@
import { LessThan } from "typeorm";
import { AppDataSource } from "./data_source.js";
import Logging from "./log.js";
import { Feed } from "./models/Feed.js";
import { createHash } from "crypto";
import rss from "rss-parser";
import { Post } from "./models/Post.js";
import bot from "./telegram.js";
interface RSSFeed {
title: string;
language: string;
description: string;
copyright: string;
items: RSSFeedItem[];
}
interface RSSFeedItem {
title: string;
link: string;
content: string,
contentSnippet: string;
guid: string;
}
function calculateHash(item: RSSFeedItem) {
let hash = createHash("sha512");
if (item.content)
hash.update(item.content);
if (item.guid)
hash.update(item.guid);
if (item.title)
hash.update(item.title);
if (item.link)
hash.update(item.link);
if (item.contentSnippet)
hash.update(item.contentSnippet);
return hash.digest("hex");
}
async function checkFeed(feed: Feed) {
Logging.info("Fetching feed: %s", feed.url);
let data = await fetch(feed.url).then(res => res.text());
Logging.info("Received Data");
Logging.debug(data);
const parser = new rss();
Logging.info("Parsing feed");
let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as RSSFeed;
Logging.info("Parsed feed");
Logging.info("Checking for new items");
for (let entry of feedData.items) {
let hash = calculateHash(entry);
let post = await AppDataSource.manager.findOne(Post, {
where: {
feed,
hash
}
});
if (!post) {
post = AppDataSource.manager.create(Post, {
feed,
hash
})
const decode = (inp) => inp;
feed.subscriber.map(async sub => {
bot.telegram.sendMessage(
sub.chatid,
entry.guid + "\n" + decode(entry.title) + "\n\n" + decode(entry.contentSnippet)
).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
})
await AppDataSource.manager.save(post);
}
}
// Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
// const newItems = feedData.items.filter(item => !feed.itemExists(item));
// // Sending notifications
// await sendFeedTelegraf(feed, newItems);
// feed.addItems(newItems);
}
// const FEED_CHECK_INTERVAL = 1000 * 60 * 60;
const FEED_CHECK_INTERVAL = 1000 * 30;
export default async function checkFeeds() {
while (true) {
let feed = await AppDataSource.manager.findOne(Feed, {
where: {
lastCheck: LessThan(new Date(Date.now() - FEED_CHECK_INTERVAL))
},
relations: {
subscriber: true
}
})
if (!feed) {
await new Promise(y => setTimeout(y, 1000));
continue;
}
await checkFeed(feed);
feed.lastCheck = new Date();
await AppDataSource.manager.save(feed);
}
}

View File

@ -1,230 +1,18 @@
// require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create();
// require('ssl-root-cas').inject();
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
import * as _Logging from "@hibas123/nodelogging";
import { createHash } from "crypto";
import * as dotenv from "dotenv";
import { decode } from "html-entities";
import { AppDataSource, appDataSourceReady } from "./data_source.js";
import checkFeeds from "./feed_fetcher.js";
import Logging from "./log.js";
import { User } from "./models/User.js";
import bot from "./telegram.js";
import fetch from "node-fetch";
import rss from "rss-parser";
import { Telegraf } from "telegraf";
import { message } from "telegraf/filters";
import * as fs from "fs";
import lodash from 'lodash'
const Logging = (_Logging.default as any).default as typeof _Logging.default;
Promise.resolve().then(async () => {
const { Low, LowSync } = await import("lowdb");
// @ts-ignore
const { JSONFileSync } = await import("lowdb/node") as any;
class LowWithLodash<T> extends LowSync<T> {
chain: lodash.ExpChain<this['data']> = lodash.chain(this).get('data')
}
dotenv.config();
const parser = new rss();
// const entities = new AllHtmlEntities();
if (!fs.existsSync("./persist")) {
fs.mkdirSync("./persist")
}
// @ts-ignore
const adapter = new JSONFileSync<{ feeds: IDBFeed[] }>("persist/db.json");
const db: any = new LowWithLodash(adapter);
db.read();
db.chain.defaults({ feeds: [] });
db.write();
interface IDBFeed {
url: string;
subscriber: number[];
oldEntries: string[];
}
class Database {
static findFeed(url: string): DBFeed {
const feed = db.chain.get("feeds").find(e => e.url === url).value();
return feed ? new DBFeed(feed) : undefined;
}
static addFeed(url: string): IDBFeed {
const feed = {
url: url,
oldEntries: [],
subscriber: []
};
db.chain.get("feeds").unshift(feed)
db.write();
return feed;
}
static addSubscriber(url: string, chatid: number) {
const feed = this.findFeed(url);
if (!feed)
this.addFeed(url);
else {
if (feed.subscriber.some(e => e === chatid)) {
return;
}
}
db.chain.get("feeds").find(e => e.url === url).get("subscriber").push(chatid)
db.write();
}
static findSubscribed(chatid: number) {
return db.chain.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value();
}
static removeSubscriber(url: string, chatid: number) {
db.chain.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid)
db.write();
}
static addItems(url: string, hashes: string[]) {
db.chain.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes)
db.write();
}
static getAll() {
return db.chain.get("feeds").map(feed => new DBFeed(feed)).value();
}
}
class DBFeed implements IDBFeed {
url: string;
subscriber: number[];
oldEntries: string[];
constructor(dbobject?: IDBFeed) {
if (dbobject) {
for (let key in dbobject)
this[key] = dbobject[key];
}
}
itemExists(item: IFeedItem): boolean {
const hash = calculateHash(item);
return !!this.oldEntries.find(e => e === hash);
}
addItems(items: IFeedItem[]) {
Database.addItems(this.url, items.map(item => calculateHash(item)));
}
}
interface Feed {
title: string;
language: string;
description: string;
copyright: string;
items: IFeedItem[];
}
interface IFeedItem {
title: string;
link: string;
content: string,
contentSnippet: string;
guid: string;
}
function calculateHash(item: IFeedItem) {
let hash = createHash("sha512");
if (item.content)
hash.update(item.content);
if (item.guid)
hash.update(item.guid);
if (item.title)
hash.update(item.title);
if (item.link)
hash.update(item.link);
if (item.contentSnippet)
hash.update(item.contentSnippet);
return hash.digest("hex");
}
async function checkFeed(feed: DBFeed) {
Logging.log("Fetching:", feed.url);
let data = await fetch(feed.url).then(res => res.text());
Logging.log("Received Data");
let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed;
// Check for new items
Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
const newItems = feedData.items.filter(item => !feed.itemExists(item));
// Sending notifications
await sendFeedTelegraf(feed, newItems);
feed.addItems(newItems);
}
const bot = new Telegraf(process.env.TG_TOKEN)
bot.start(async ctx => {
await ctx.reply("Send some RSS Feed URLs to get started.");
})
bot.command("list", async (ctx) => {
const chatid = ctx.chat.id;
const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n");
await ctx.reply("You are currently subscribed to: \n" + feeds);
})
bot.on(message("text", "entities"), async ctx => {
const chatid = ctx.chat.id;
Logging.debug("Message From:", chatid, ctx.message);
const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length));
await Promise.all(urls.map(async url => {
Database.addSubscriber(url, chatid);
await ctx.reply("Subscribed to: " + url);
let feed = Database.findFeed(url)
if (feed)
checkFeed(feed);
else
Logging.error("Cannot find created feed!")
}))
if (urls.length === 0)
await ctx.reply("No URLs found in message.");
})
bot.launch()
async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) {
Logging.debug("Before send", feed, items);
await Promise.all(feed.subscriber.map(
subscriber => Promise.all(
items.map(
item => bot.telegram.sendMessage(
subscriber,
item.guid + "\n" + decode(item.title) + "\n\n" + decode(item.contentSnippet)
).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
)
)
));
}
function checkAll() {
Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)).finally(() => {
db.write();
}));
}
setInterval(() => {
checkAll();
}, 1000 * 60 * 60);
checkAll();
})
appDataSourceReady.then(() => {
return Promise.allSettled([
bot.launch({}),
checkFeeds()
])
}).then(() => {
Logging.info("Exiting...");
}).catch(err => {
Logging.error(err);
process.exit(1);
});

12
src/log.ts Normal file
View File

@ -0,0 +1,12 @@
import pino from "pino";
const Logging = pino({
transport: {
target: "pino-pretty",
options: {
colorize: true
}
}
})
export default Logging;

0
src/migrate.ts Normal file
View File

24
src/models/Feed.ts Normal file
View File

@ -0,0 +1,24 @@
import { Entity, Unique, PrimaryGeneratedColumn, Column, ManyToMany, JoinTable, OneToMany, Relation } from "typeorm";
import { User } from "./User.js";
import { Post } from "./Post.js";
@Entity()
export class Feed {
@PrimaryGeneratedColumn()
id: number;
@Column()
url: string;
@Column({
default: "1970-01-01 01:00:00.000"
})
lastCheck?: Date;
@ManyToMany(() => User, user => user.feeds)
@JoinTable()
subscriber: Relation<User>[];
@OneToMany(() => Post, post => post.feed)
oldEntries: Relation<Post>[];
}

11
src/models/Post.ts Normal file
View File

@ -0,0 +1,11 @@
import { PrimaryColumn, Entity, ManyToOne, Relation } from "typeorm";
import { Feed } from "./Feed.js";
@Entity()
export class Post {
@PrimaryColumn()
hash: string;
@ManyToOne(() => Feed, (feed) => feed.oldEntries)
feed: Relation<Feed>;
}

15
src/models/User.ts Normal file
View File

@ -0,0 +1,15 @@
import { PrimaryGeneratedColumn, ManyToMany, Relation, Entity, Column, Unique } from "typeorm"
import { Feed } from "./Feed.js";
@Entity()
@Unique("chatid", ["chatid"])
export class User {
@PrimaryGeneratedColumn()
id: number;
@Column()
chatid: number;
@ManyToMany(() => Feed, feed => feed.subscriber)
feeds: Relation<Feed>[];
}

230
src/old_index.ts Normal file
View File

@ -0,0 +1,230 @@
// require('https').globalAgent.options.ca = require('ssl-root-cas/latest').create();
// require('ssl-root-cas').inject();
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";
import * as _Logging from "@hibas123/nodelogging";
import { createHash } from "crypto";
import * as dotenv from "dotenv";
import { decode } from "html-entities";
import fetch from "node-fetch";
import rss from "rss-parser";
import { Telegraf } from "telegraf";
import { message } from "telegraf/filters";
import * as fs from "fs";
import lodash from 'lodash'
const Logging = (_Logging.default as any).default as typeof _Logging.default;
Promise.resolve().then(async () => {
const { Low, LowSync } = await import("lowdb");
// @ts-ignore
const { JSONFileSync } = await import("lowdb/node") as any;
class LowWithLodash<T> extends LowSync<T> {
chain: lodash.ExpChain<this['data']> = lodash.chain(this).get('data')
}
dotenv.config();
const parser = new rss();
// const entities = new AllHtmlEntities();
if (!fs.existsSync("./persist")) {
fs.mkdirSync("./persist")
}
// @ts-ignore
const adapter = new JSONFileSync<{ feeds: IDBFeed[] }>("persist/db.json");
const db: any = new LowWithLodash(adapter);
db.read();
db.chain.defaults({ feeds: [] });
db.write();
interface IDBFeed {
url: string;
subscriber: number[];
oldEntries: string[];
}
class Database {
static findFeed(url: string): DBFeed {
const feed = db.chain.get("feeds").find(e => e.url === url).value();
return feed ? new DBFeed(feed) : undefined;
}
static addFeed(url: string): IDBFeed {
const feed = {
url: url,
oldEntries: [],
subscriber: []
};
db.chain.get("feeds").unshift(feed)
db.write();
return feed;
}
static addSubscriber(url: string, chatid: number) {
const feed = this.findFeed(url);
if (!feed)
this.addFeed(url);
else {
if (feed.subscriber.some(e => e === chatid)) {
return;
}
}
db.chain.get("feeds").find(e => e.url === url).get("subscriber").push(chatid)
db.write();
}
static findSubscribed(chatid: number) {
return db.chain.get("feeds").filter(e => e.subscriber.indexOf(chatid) >= 0).value();
}
static removeSubscriber(url: string, chatid: number) {
db.chain.get("feeds").find(e => e.url === url).get("subscriber").remove(e => e === chatid)
db.write();
}
static addItems(url: string, hashes: string[]) {
db.chain.get("feeds").find(e => e.url === url).get("oldEntries").unshift(...hashes)
db.write();
}
static getAll() {
return db.chain.get("feeds").map(feed => new DBFeed(feed)).value();
}
}
class DBFeed implements IDBFeed {
url: string;
subscriber: number[];
oldEntries: string[];
constructor(dbobject?: IDBFeed) {
if (dbobject) {
for (let key in dbobject)
this[key] = dbobject[key];
}
}
itemExists(item: IFeedItem): boolean {
const hash = calculateHash(item);
return !!this.oldEntries.find(e => e === hash);
}
addItems(items: IFeedItem[]) {
Database.addItems(this.url, items.map(item => calculateHash(item)));
}
}
interface Feed {
title: string;
language: string;
description: string;
copyright: string;
items: IFeedItem[];
}
interface IFeedItem {
title: string;
link: string;
content: string,
contentSnippet: string;
guid: string;
}
function calculateHash(item: IFeedItem) {
let hash = createHash("sha512");
if (item.content)
hash.update(item.content);
if (item.guid)
hash.update(item.guid);
if (item.title)
hash.update(item.title);
if (item.link)
hash.update(item.link);
if (item.contentSnippet)
hash.update(item.contentSnippet);
return hash.digest("hex");
}
async function checkFeed(feed: DBFeed) {
Logging.log("Fetching:", feed.url);
let data = await fetch(feed.url).then(res => res.text());
Logging.log("Received Data");
let feedData = await parser.parseString(data).catch(err => Logging.error(err)) as Feed;
// Check for new items
Logging.debug(feedData.items.length, feedData.items.map(e => calculateHash(e)));
const newItems = feedData.items.filter(item => !feed.itemExists(item));
// Sending notifications
await sendFeedTelegraf(feed, newItems);
feed.addItems(newItems);
}
const bot = new Telegraf(process.env.TG_TOKEN)
bot.start(async ctx => {
await ctx.reply("Send some RSS Feed URLs to get started.");
})
bot.command("list", async (ctx) => {
const chatid = ctx.chat.id;
const feeds = Database.findSubscribed(chatid).map(feed => feed.url).join("\n");
await ctx.reply("You are currently subscribed to: \n" + feeds);
})
bot.on(message("text", "entities"), async ctx => {
const chatid = ctx.chat.id;
Logging.debug("Message From:", chatid, ctx.message);
const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.substr(e.offset, e.length));
await Promise.all(urls.map(async url => {
Database.addSubscriber(url, chatid);
await ctx.reply("Subscribed to: " + url);
let feed = Database.findFeed(url)
if (feed)
checkFeed(feed);
else
Logging.error("Cannot find created feed!")
}))
if (urls.length === 0)
await ctx.reply("No URLs found in message.");
})
bot.launch()
async function sendFeedTelegraf(feed: DBFeed, items: IFeedItem[]) {
Logging.debug("Before send", feed, items);
await Promise.all(feed.subscriber.map(
subscriber => Promise.all(
items.map(
item => bot.telegram.sendMessage(
subscriber,
item.guid + "\n" + decode(item.title) + "\n\n" + decode(item.contentSnippet)
).catch(err => Logging.error(err)).then(() => Logging.debug("Message Sent"))
)
)
));
}
function checkAll() {
Database.getAll().map(feed => checkFeed(feed).catch(err => Logging.error(err)).finally(() => {
db.write();
}));
}
setInterval(() => {
checkAll();
}, 1000 * 60 * 60);
checkAll();
})

162
src/telegram.ts Normal file
View File

@ -0,0 +1,162 @@
import { Markup, Telegraf } from "telegraf";
import { message } from "telegraf/filters";
import { AppDataSource } from "./data_source.js";
import { User } from "./models/User.js";
import Logging from "./log.js";
import { Feed } from "./models/Feed.js";
const bot = new Telegraf(process.env.TG_TOKEN)
function botHandler<T>(handler: (ctx: T) => Promise<any>) {
return async (ctx: T) => {
try {
await handler(ctx);
} catch (err) {
Logging.error(err);
(ctx as any).reply("An error occured while processing your request.");
}
}
}
bot.start(botHandler(async ctx => {
await ctx.reply("Send some RSS Feed URLs to get started.");
}));
bot.command("list", botHandler(async (ctx) => {
let user = await AppDataSource.manager.findOne(User, {
where: {
chatid: ctx.chat.id
},
relations: {
feeds: true,
}
})
if (!user) {
ctx.reply("You have no feeds yet.");
} else {
ctx.reply("You are currently subscribed to: \n" + user.feeds.map(feed => feed.url).join("\n"));
}
}));
bot.command("delete", botHandler(async ctx => {
let user = await AppDataSource.manager.findOne(User, {
where: {
chatid: ctx.chat.id
},
relations: {
feeds: true,
}
})
if (!user || user.feeds.length === 0) {
ctx.reply("You have no feeds yet.");
} else {
ctx.replyWithHTML("Select a feed to delete:", {
...Markup.inlineKeyboard(user.feeds.map(feed => Markup.button.callback(feed.url, "delete:" + feed.id)))
})
}
}));
bot.on("callback_query", botHandler(async ctx => {
Logging.info("Callback Query:");
Logging.info(ctx.callbackQuery);
const remove_options = () => {
ctx.editMessageReplyMarkup({
inline_keyboard: []
})
}
let data = (ctx.callbackQuery as any).data;
if (!data) {
ctx.answerCbQuery("Invalid data");
remove_options();
return
}
let [action, id] = data.split(":");
if (!action || !id) {
ctx.answerCbQuery("Invalid data");
remove_options();
return
}
if (action === "delete") {
let user = await AppDataSource.manager.findOne(User, {
where: {
chatid: ctx.chat.id
},
relations: {
feeds: true,
}
});
if (user) {
user.feeds = user.feeds.filter(feed => feed.id !== Number(id));
await AppDataSource.manager.save(user);
}
} else {
ctx.answerCbQuery("Invalid action");
remove_options();
return
}
ctx.answerCbQuery("Success");
remove_options();
}));
bot.on(message("text", "entities"), botHandler(async ctx => {
let user = await AppDataSource.manager.findOne(User, {
where: {
chatid: ctx.chat.id
}
});
if (!user) {
user = AppDataSource.manager.create(User, {
chatid: ctx.chat.id,
feeds: [],
})
await AppDataSource.manager.save(user);
}
Logging.debug("Received message:", ctx.message);
const urls = ctx.message.entities.filter(e => e.type === "url").map(e => ctx.message.text.slice(e.offset, e.offset + e.length));
Logging.debug("URLs:", urls);
for (let url of urls) {
let normalized = new URL(url).href;
Logging.debug("Normalized URL:", normalized);
let feed = await AppDataSource.manager.findOne(Feed, {
where: {
url: normalized
},
relations: {
subscriber: true,
}
});
if (feed) {
feed.subscriber.push(user);
await AppDataSource.manager.save(feed);
} else {
feed = AppDataSource.manager.create(Feed, {
oldEntries: [],
subscriber: [user],
url: normalized,
})
await AppDataSource.manager.save(feed);
}
}
await ctx.reply("Added feeds.");
}));
export default bot;

View File

@ -8,7 +8,9 @@
"esModuleInterop": true,
"preserveWatchOutput": true,
"forceConsistentCasingInFileNames": true,
"skipLibCheck": true
"skipLibCheck": true,
"experimentalDecorators": true,
"emitDecoratorMetadata": true
},
"include": ["./src"],
"exclude": ["node_modules"]

1837
yarn.lock

File diff suppressed because it is too large Load Diff