first commit

This commit is contained in:
peertube 2020-12-09 21:19:58 +01:00
commit da6d35b2e7
17 changed files with 2442 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
node_modules
logs/*.log

5
conf/discord.json Normal file
View File

@ -0,0 +1,5 @@
{
"clientID": "701077243652341762",
"token": "NzAxMDc3MjQzNjUyMzQxNzYy.XpsPmw.luwV8qmRlYwibpA3hzT9SnN5mvk",
"secret": "oxiy0FoaI2e4ni0lkK6B9yqaS5IOJ2e1"
}

25
conf/routes.json Normal file
View File

@ -0,0 +1,25 @@
{
"discordBot": {},
"rss-youtube": {
"discord": {
"channel": "701093105864147024",
"color": "#FF0000",
"style": "tinyEmbed",
"footer": {
"text": "vidéo Youtube",
"icon_url": "https://data.kataku.in/cdn/youtube_social.png"
}
}
},
"rss-skeptikon": {
"discord": {
"channel": "701093136713252925",
"color": "#F1680D",
"style": "embed",
"footer": {
"text": "vidéo Peertube",
"icon_url": "https://data.kataku.in/cdn/skeptikon_social.png"
}
}
}
}

15
conf/skepticom.json Normal file
View File

@ -0,0 +1,15 @@
{
"services" : [
"logs",
"discord",
"rss-skeptikon",
"rss-youtube"
],
"pm2": {
"prefix": "skp_",
"mode": "cluster"
},
"logs": {
"path": "./logs"
}
}

9
db/rss-skeptikon.json Normal file
View File

@ -0,0 +1,9 @@
{
"config": {
"timeLoop": 5,
"urlFeeds": "https://skeptikon.fr/feeds/videos.xml?sort=-publishedAt&filter=local",
"sliceDescription": 142
},
"lastCheck": "2020-12-09T20:12:27.226Z",
"latest": "2020-12-09T13:22:59.000Z"
}

62
db/rss-youtube.json Normal file
View File

@ -0,0 +1,62 @@
{
"config": {
"timeLoop": 5,
"urlFeeds": "https://www.youtube.com/feeds/videos.xml?channel_id=",
"sliceDescription": 142
},
"feeds": {
"Astroscept": "UCWX5WRYYAFyvgL8vTV8FT8g",
"Apéros sceptiques de Lyon": "UC5ew8MK-qfzYtZo1T3ysuVA",
"Chat Sceptique": "UCOuIgj0CYCXCvjWywjDbauw",
"Clément FREZE": "UCY7klexd1qEqxgqYK6W7BVQ",
"Curiosity Catalyst": "UCg47pIhBRJbz52-ABnm0uxw",
"Édukey": "UC6wF0stG_iXMO4mIXM9g3YQ",
"Game of Hearth": "UC3A_TG1leX0eQEJD1Ew6Ftw",
"Gollum Illuminati": "UCGHez4gV7nwUcS42ihkNnZg",
"Histony": "UCt8ctlakIflnSG0ebFps7cw",
"Hygiène Mentale": "UCMFcMhePnH4onVHt2-ItPZw",
"Jaco patrolman": "UCx7Lo8n7Y7lf-FXC6B7kNBQ",
"Ju [kataku]": "UCccKnlXUoCq31DRn4uE_SbA",
"L'extracteur": "UCvtMQXNgBROo1Sh2D3Qiulw",
"Le corbeau qui croit": "UCvwFa6G3ZR7OpmsCI5zYNQw",
"Le réveilleur": "UC1EacOJoqsKaYxaDomTCTEQ",
"Mi-fougue, mi-raison": "UCvOMVsh8ks3Ix0LW5kUd_TA",
"Mr.Sam - Point d'interrogation": "UCh2YBKhYIy-_LtfCIn2Jycg",
"Méta de Choc": "UCeYerYR680Q56i4D1HJcF8Q",
"Passé Sauvage": "UCLhOJJbPciPdocXTaAk2SdA",
"Projet Utopia": "UCoqAyMfi7YZUFw3L8WumcLA",
"Spline LND": "UCLdmnkqdcTPHvVZ8aNdbf5A",
"Temps Mort": "UCJLsfsSpIiUBPvYYSUAclrw",
"Tranxen": "UCLRiUYs2x5FhLWXRS5138YQ",
"Tzitzimitl": "UC0yPCUmdMZIGtnxSnx5_ifA",
"Un Monde Riant": "UC8T_vTz76WUsudvxPk6SLEw"
},
"timecode": {
"lastCheck": "2020-12-09T20:12:31.950Z",
"Un Monde Riant": "2020-05-25T18:07:01.000Z",
"Méta de Choc": "2020-12-04T17:00:18.000Z",
"Ju [kataku]": "2020-04-01T00:00:00.923Z",
"Temps Mort": "2020-05-10T18:00:11.000Z",
"Spline LND": "2020-10-30T16:02:19.000Z",
"Chat Sceptique": "2020-12-01T17:30:03.000Z",
"Projet Utopia": "2020-10-20T16:00:02.000Z",
"Curiosity Catalyst": "2020-04-01T00:00:00.923Z",
"Le corbeau qui croit": "2020-04-01T00:00:00.923Z",
"Mi-fougue, mi-raison": "2020-04-01T00:00:00.923Z",
"Mr.Sam - Point d'interrogation": "2020-12-01T17:31:38.000Z",
"Histony": "2020-10-29T16:30:49.000Z",
"Hygiène Mentale": "2020-11-13T08:00:11.000Z",
"Gollum Illuminati": "2020-11-10T22:46:45.000Z",
"Tranxen": "2020-11-27T16:00:27.000Z",
"Passé Sauvage": "2020-11-27T12:28:49.000Z",
"Clément FREZE": "2020-09-09T14:31:28.000Z",
"Tzitzimitl": "2020-11-29T23:46:15.000Z",
"Astroscept": "2020-12-01T07:51:57.000Z",
"Apéros sceptiques de Lyon": "2020-11-14T17:44:11.713Z",
"Édukey": "2020-11-14T17:44:11.713Z",
"Game of Hearth": "2020-12-09T18:04:26.000Z",
"Jaco patrolman": "2020-11-28T21:29:24.000Z",
"L'extracteur": "2020-12-09T13:00:33.000Z",
"Le réveilleur": "2020-11-14T17:44:11.713Z"
}
}

172
index.js Normal file
View File

@ -0,0 +1,172 @@
/*
SkeptCOM | Ceci est l'index, c'est ici qu'on va load les process PM2
*/
// CONF
const skepticonf = require('./conf/skepticom.json')
// REQUIRE
const pm2 = require('pm2')
const path = require('path')
// FUNCTIONS
function buildAppName(name) {
return {
"name" : skepticonf.pm2.prefix + name,
"script" : "./services/" + name + ".js",
"cwd" : path.resolve(__dirname),
"instances" : 1,
"exec_mode" : skepticonf.pm2.mode,
"args" : [ "--color" ],
"watch" : false,
"ignore_watch" : ["node_modules", "conf", "db"],
"watch_options": {
"followSymlinks": true,
"persistent" : true,
"ignoreInitial" : true
}
}
}
// INIT
if (process.argv[2] && process.argv[2] === '--service') {
let service = process.argv[3]
let action = process.argv[4]
let options = process.argv.slice(5, process.argv.length)
// console.log(`[coucou toi]`, service, action, options) // DEV ONLY
if (service == 'list') {
pm2.list((err, list) => {
if (err) return console.log(`ERROR pm2.list()`)
if (list.length == 0) return console.log(` -- NO SERVICE RUNNING`)
for (let i in list) {
console.log(` -- SERVICE ${list[i].name.replace(skepticonf.pm2.prefix, '')} [WORKING] [${Number(i)+1}/${list.length}] `)
}
pm2.disconnect()
})
} else if (!service || !action) {
return console.error(`[ERROR] '--service NomDuService ActionDuService'`)
} else {
if (action == 'stop') {
pm2.list((err, list) => {
if (err) return console.error(`[ERROR:1.1] '--service NomDuService stop'`)
let servDesc
for (let i in list) {
if (list[i].name.replace(skepticonf.pm2.prefix, '') === service) {
servDesc = list[i]
}
}
if (servDesc != undefined) {
pm2.connect(err => {
if (err) console.error(err)
pm2.delete(servDesc.name, err => {
if (err) return console.error(`[ERROR:1.2] '--service NomDuService stop'`)
console.log(` -- SERVICE ${servDesc.name} [STOP]`)
pm2.disconnect()
})
})
} else {
console.log(` [INFO] This service is not load, Check '--service list' for more information`)
}
pm2.disconnect()
})
} else if (action === 'start') {
if(!skepticonf.services.includes(service)) return console.log(` [INFO] This service does not exist`)
pm2.list((err, list) => {
if (err) return console.error(`[ERROR:2.1] '--service NomDuService start'`)
let servDesc
for (let i in list) {
if (list[i].name.replace(skepticonf.pm2.prefix, '') === service) {
servDesc = list[i]
}
}
if (servDesc === undefined) {
let appName = buildAppName(service)
pm2.connect(err => {
if (err) console.error(err)
pm2.start(appName, err => {
if (err) return console.error(`[ERROR:2.2] '--service NomDuService start'`)
console.log(` -- SERVICE ${service} [LOAD]`)
pm2.disconnect()
})
})
} else {
console.log(` [INFO] This service is load, Check '--service list' for more information`)
}
pm2.disconnect()
})
} else if (action === 'restart') {
if(!skepticonf.services.includes(service)) return console.log(` [INFO] This service does not exist`)
pm2.list((err, list) => {
if (err) return console.error(`[ERROR:3.1] '--service NomDuService restart'`)
let servDesc
for (let i in list) {
if (list[i].name.replace(skepticonf.pm2.prefix, '') === service) {
servDesc = list[i]
}
}
if (servDesc != undefined) {
let appName = buildAppName(service)
pm2.connect(err => {
if (err) console.error(err)
pm2.restart(appName, err => {
if (err) return console.error(`[ERROR:3.2] '--service NomDuService restart'`)
console.log(` -- SERVICE ${service} [RELOAD]`)
pm2.disconnect()
})
})
} else {
console.log(` [INFO] This service is not load, Check '--service list' for more information`)
}
pm2.disconnect()
})
}
}
} else {
pm2.connect(err => {
if (err) console.error(err)
for (let i in skepticonf.services) {
let appName = buildAppName(skepticonf.services[i])
if (process.argv[2] === "stop") {
pm2.delete(appName.name, err => {
if (err) return console.log(` -- SERVICE ${skepticonf.services[i]} [UNLOADED] [${Number(i)+1}/${skepticonf.services.length}]`)
console.log(` -- SERVICE ${skepticonf.services[i]} [STOP] [${Number(i)+1}/${skepticonf.services.length}]`)
})
} else {
pm2.start(appName, err => {
if (err) return console.error(err)
console.log(` -- SERVICE ${skepticonf.services[i]} [LOAD] [${Number(i)+1}/${skepticonf.services.length}]`)
})
}
if (i == (skepticonf.services.length-1)) {
setTimeout(function(){
console.log(` -- ALL PROCESS FINISH -- \n`)
pm2.disconnect()
process.exit()
}, 2000)
}
}
})
}

0
logs/.git-keep Normal file
View File

41
modules/discordBot.js Normal file
View File

@ -0,0 +1,41 @@
/*
SkeptCOM [Discord] | Pour envoyer des notifications sur un serveur Discord
*/
// LOAD configFiles
const discordConf = require('../conf/discord.json')
const routage = require('../modules/routage')('discordBot')
// REQUIRE, CONST, VARIABLES
const Discord = require('discord.js')
const client = new Discord.Client()
// FUNCTIONS
const myDiscord = {
send: function() {}
}
// INIT
client.on('ready', () => {
console.log(` --- [Discord] Logged in as ${client.user.tag}!`)
routage.send(` --- [Discord] Logged in as ${client.user.tag}!`)
myDiscord.send = (channel, msg) => {
client.channels.fetch(channel)
.then(channel => channel.send(msg))
.catch(console.error)
}
})
// client.on('message', message => {
// if (message.content.startsWith('!ping')) {
// console.log(`on a un ping`)
// routage.send(message.content)
// }
// })
client.login(discordConf.token)
module.exports = myDiscord

30
modules/routage.js Normal file
View File

@ -0,0 +1,30 @@
/*
SkeptCOM [IO] | Via un json, on va rediriger qui envoye a quoi
*/
const routes = require('../conf/routes.json')
const skepticonf = require('../conf/skepticom.json')
const services = {}
module.exports = (who) => {
if (!who || !routes[who]) return console.error(`ERROR routage`)
services[who] = require('./skeptIO')(who)
function send(data) {
services[who].send('logs', '', data)
for (let i in routes[who]) {
services[who].send(i, routes[who][i], data)
}
}
function log(data) {
services[who].send('logs', '', data)
}
return {
send,
log
}
}

40
modules/skeptIO.js Normal file
View File

@ -0,0 +1,40 @@
/*
SkeptIO | Gestionnaire des services
*/
const pm2 = require('pm2')
module.exports = (who) => {
function sendMSG(to, conf, data) {
if (!who) return console.error(`ERROR skeptIO ${to}`)
let dataMSG = {
topic: to,
from: 'skp_' + who,
conf,
data
}
pm2.list( (err, list) => { // on check la liste des process pm2
for (let i in list) {
if (list[i].name === 'skp_' + to) { // on verif que le process exite bien
pm2.sendDataToProcessId(list[i].pm_id, dataMSG, (err, res) => { // et on send
if (err) return console.error(` <-- [${list[i].pm_id}:${list[i].name}] error`, err)
})
}
}
})
}
function receiveMSG(res) {
process.on('message', data => {
res(data)
})
}
return {
send: sendMSG,
receive: receiveMSG
}
}

1752
package-lock.json generated Normal file

File diff suppressed because it is too large Load Diff

22
package.json Normal file
View File

@ -0,0 +1,22 @@
{
"name": "SkeptiCOM",
"version": "0.2.10",
"description": "Bot Skeptikon",
"main": "index.js",
"scripts": {
"start": "node index.js",
"service": "node index.js --service",
"stop": "node index.js stop"
},
"author": "Ju [kataku] / Skeptikon",
"dependencies": {
"colors": "^1.4.0",
"dateformat": "^3.0.3",
"discord.js": "^12.0.1",
"fs-extra": "^9.0.0",
"jsonfile": "^6.0.1",
"lodash": "^4.17.15",
"pm2": "^4.2.3",
"rss-parser": "^3.7.5"
}
}

54
services/discord.js Normal file
View File

@ -0,0 +1,54 @@
/*
SkeptCOM [Discord] | Pour envoyer des notifications sur un serveur Discord
exemple pour injecter depuis un routage :
routage.send({
author: "",
title: "",
url : "",
image: "",
desc: ""
})
*/
// REQUIRE, CONST, VARIABLES
const discordBot = require('../modules/discordBot')
const skeptIO = require('../modules/skeptIO')('discord')
function embed(conf, data) {
return {
embed: {
color: conf.color,
author: {name: data.author},
title: data.title,
url: data.url,
image: {url: data.image },
description: data.desc,
footer: conf.footer
}
}
}
function tinyEmbed(conf, data) {
return {
embed: {
color: conf.color,
author: {name: data.author},
title: data.title,
url: data.url,
thumbnail: {url: data.image},
description: data.desc,
footer: conf.footer
}
}
}
// INIT
skeptIO.receive( obj => {
console.log(` --> [Discord] message from [${obj.from}]`)
if (obj.conf.style === "msg") discordBot.send(obj.conf.channel, obj.data.desc)
if (obj.conf.style === "embed") discordBot.send(obj.conf.channel, embed(obj.conf, obj.data))
if (obj.conf.style === "tinyEmbed") discordBot.send(obj.conf.channel, tinyEmbed(obj.conf, obj.data))
})

35
services/logs.js Normal file
View File

@ -0,0 +1,35 @@
/*
SkeptCOM [logs] | on va log les choses, c'est mieux quand on a un problème ... 'fin, les gens bons le fond souvent
*/
// REQUIRE, CONST, VARIABLES
const dateFormat = require('dateformat')
const fs = require('fs')
const fse = require('fs-extra')
const colors = require('colors')
const path = require('path')
const skepticonf = require('../conf/skepticom.json')
const skeptIO = require('../modules/skeptIO')('logs')
const logConf = {}
// INIT
if (!skepticonf.logs) return console.log(` [ERROR || LOGS] no configuration for logs.js in ./conf/skepticom.json`)
if (skepticonf.logs.path) {
logConf.path = path.resolve(skepticonf.logs.path)
} else {
logConf.path = path.resolve('./_logs')
}
fse.ensureDirSync(logConf.path)
skeptIO.receive( obj => {
let rtn = `[${dateFormat("yyyy-mm-dd HH:MM")}] ${obj.from.replace(skepticonf.pm2.prefix, '')}\n ${JSON.stringify(obj.data)}`
console.log(rtn.magenta)
fs.appendFile(logConf.path + '/' + dateFormat("yyyy-W") + '.log', rtn + '\n', 'utf8', function(err) {
if (err) return console.error(err)
})
})
console.log(` --- [LOGS] Load`)

87
services/rss-skeptikon.js Normal file
View File

@ -0,0 +1,87 @@
/*
SkeptCOM [RSS-Youtube] | Détecte les nouvelles vidéos YT via RSS
*/
// REQUIRE, CONST, VARIABLES
const jsonfile = require('jsonfile')
const Parser = require('rss-parser')
const path = require('path')
const routage = require('../modules/routage')('rss-skeptikon')
let parser = new Parser({
headers: {
Accept: 'application/rss+xml, application/xml',
}
})
const db = require('../db/rss-skeptikon.json')
// FUNCTIONS
// et on output ! YOLO !
function output(data) {
console.log(` --> [RSS-Skeptikon | NV] ${data.author} | ${data.title}`)
if (routage && routage.send) routage.send(data)
}
// bon, on a besoin de recup le RSS, normol
async function getFeed(url) {
return new Promise((resolve, reject) => {
parser.parseURL(url, (err, res) => {
if (err) return reject(err)
resolve(res)
})
})
}
// on filtre un peu, puis en fonction on output
async function parseRSS(feed) {
for (let n in feed.items) {
let i = feed.items.length-n-1 // on check en décroissant
if (feed.items[i].isoDate > db.lastCheck || feed.items[i].isoDate > db.latest) {
db.latest = feed.items[i].isoDate
let id = feed.items[i].link.split('/')[feed.items[i].link.split('/').length-1]
let c = {
author: feed.items[i].creator,
title: feed.items[i].title,
url: feed.items[i].link,
id: id,
image: "https://skeptikon.fr/static/previews/" + id + ".jpg",
desc: feed.items[i].content.slice(0, db.config.sliceDescription) + " [...]",
isoDate: feed.items[i].isoDate
}
output(c)
}
}
}
// on check l'ensemble de feeds
async function checkFeeds(feeds) {
console.log(` --- [RSS-Skeptikon] check @${db.lastCheck}`)
routage.log(`check @${db.lastCheck}`)
let feed = await getFeed(feeds)
await parseRSS(feed)
let dateObj = new Date()
db.lastCheck = dateObj.toISOString()
// et on externalise la nouvelle date. Youpi !
jsonfile.writeFile(path.resolve('./db/rss-skeptikon.json'), db, {spaces: 2}, (err) => {
if (err) console.error(err)
})
}
function rssLoop(time) {
checkFeeds(db.config.urlFeeds)
setTimeout(() => {
rssLoop(time)
}, time)
}
// INIT
console.log(` --- [RSS-Skeptikon] Load`)
routage.log(`Load`)
rssLoop(db.config.timeLoop*60*1000)

91
services/rss-youtube.js Normal file
View File

@ -0,0 +1,91 @@
/*
SkeptCOM [RSS-Youtube] | Détecte les nouvelles vidéos YT via RSS
*/
// REQUIRE, CONST, VARIABLES
const jsonfile = require('jsonfile')
const Parser = require('rss-parser')
const path = require('path')
const routage = require('../modules/routage')('rss-youtube')
let parser = new Parser({
customFields: {
item: [ 'author', 'media:group' ]
}
})
const db = require('../db/rss-youtube.json')
// FUNCTIONS
// et on output ! YOLO !
function output(data) {
console.log(` --> [RSS-Youtube | NV] ${data.author} | ${data.title}`)
if (routage && routage.send) routage.send(data)
}
// bon, on a besoin de recup le RSS, normol
async function getFeed(url) {
return new Promise((resolve, reject) => {
parser.parseURL(url, (err, res) => {
if (err) return reject(err)
resolve(res)
})
})
}
// on filtre un peu, puis en fonction on output
async function parseRSS(feed, owner) {
if (!db.timecode[owner]) db.timecode[owner] = db.timecode.lastCheck
let cLastCheck = db.timecode[owner]
for (let i in feed.items) {
if (feed.items[i].isoDate > db.timecode.lastCheck || feed.items[i].isoDate > cLastCheck) {
if (feed.items[i].isoDate > cLastCheck) cLastCheck = feed.items[i].isoDate
let c = feed.items[i]
c.desc = feed.items[i]['media:group']['media:description'][0].slice(0, db.config.sliceDescription) + " [...]"
c.image = "http://i3.ytimg.com/vi/" + c.id.split(':')[2] + "/maxresdefault.jpg"
c.url = c.link
delete c.link
delete c['media:group']
if (c.isoDate > cLastCheck) cLastCheck = feed.items[i].isoDate
output(c)
}
}
db.timecode[owner] = cLastCheck
}
// on check l'ensemble de feeds
async function checkFeeds(feeds) {
console.log(` --- [RSS-Youtube] check @${db.timecode.lastCheck}`)
routage.log(`check @${db.timecode.lastCheck}`)
for (let i in feeds) {
let feed = await getFeed(db.config.urlFeeds + feeds[i])
try {
await parseRSS(feed, i)
} catch (e) {
console.error("error while parsing feed %s", feed, e);
}
}
let dateObj = new Date()
db.timecode.lastCheck = dateObj.toISOString()
// et on externalise la nouvelle date. Youpi !
jsonfile.writeFile(path.resolve('./db/rss-youtube.json'), db, {spaces: 2}, (err) => {
console.log()
if (err) console.error(err)
})
}
function rssLoop(time) {
checkFeeds(db.feeds)
setTimeout(() => {
rssLoop(time)
}, time)
}
// INIT
console.log(` --- [RSS-Youtube] Load`)
routage.log(`Load`)
rssLoop(db.config.timeLoop*60*1000)