feat(monorepo): migrate to typescript monorepo
This commit is contained in:
BernardNganduDev
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"fetch": {
|
||||
"async": {
|
||||
"prefix": "basango:crawler",
|
||||
"queues": {
|
||||
"details": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS)%",
|
||||
"listing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_LISTING)%",
|
||||
"processing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING)%"
|
||||
},
|
||||
"redisUrl": "%env(BASANGO_CRAWLER_ASYNC_REDIS_URL)%",
|
||||
"ttl": {
|
||||
"default": 600,
|
||||
"failure": "%env(number:BASANGO_CRAWLER_ASYNC_TTL_FAILURE)%",
|
||||
"result": "%env(number:BASANGO_CRAWLER_ASYNC_TTL_RESULT)%"
|
||||
}
|
||||
},
|
||||
"client": {
|
||||
"backoffInitial": 1,
|
||||
"backoffMax": 30,
|
||||
"backoffMultiplier": 2,
|
||||
"followRedirects": true,
|
||||
"maxRetries": "%env(number:BASANGO_CRAWLER_FETCH_MAX_RETRIES)%",
|
||||
"respectRetryAfter": "%env(boolean:BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER)%",
|
||||
"rotate": true,
|
||||
"timeout": 20,
|
||||
"userAgent": "%env(BASANGO_CRAWLER_FETCH_USER_AGENT)%",
|
||||
"verifySsl": true
|
||||
},
|
||||
"crawler": {
|
||||
"direction": "%env(BASANGO_CRAWLER_UPDATE_DIRECTION)%",
|
||||
"maxWorkers": 5,
|
||||
"notify": false,
|
||||
"useMultiThreading": false
|
||||
}
|
||||
},
|
||||
"paths": {
|
||||
"config": "%env(BASANGO_CRAWLER_CONFIG_PATH)%",
|
||||
"data": "%env(BASANGO_CRAWLER_DATA_PATH)%",
|
||||
"root": "%env(BASANGO_CRAWLER_ROOT_PATH)%"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
{
|
||||
"sources": {
|
||||
"html": [
|
||||
{
|
||||
"paginationTemplate": "actualite",
|
||||
"requiresDetails": true,
|
||||
"requiresRateLimit": false,
|
||||
"sourceDate": {
|
||||
"pattern": "/(\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
||||
"replacement": "$3-$2-$1 $4"
|
||||
},
|
||||
"sourceId": "radiookapi.net",
|
||||
"sourceKind": "html",
|
||||
"sourceSelectors": {
|
||||
"articleBody": ".field-name-body",
|
||||
"articleCategories": ".views-field-field-cat-gorie a",
|
||||
"articleDate": ".views-field-created",
|
||||
"articleLink": ".views-field-title a",
|
||||
"articles": ".view-content > .views-row.content-row",
|
||||
"articleTitle": "h1.page-header",
|
||||
"pagination": "ul.pagination > li.pager-last > a"
|
||||
},
|
||||
"sourceUrl": "https://www.radiookapi.net",
|
||||
"supportsCategories": false
|
||||
},
|
||||
{
|
||||
"categories": ["politique", "economie", "culture", "sport", "societe"],
|
||||
"paginationTemplate": "index.php/category/{category}",
|
||||
"requiresDetails": false,
|
||||
"requiresRateLimit": false,
|
||||
"sourceDate": {
|
||||
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
||||
"replacement": "$3-$2-$1 $4"
|
||||
},
|
||||
"sourceId": "7sur7.cd",
|
||||
"sourceKind": "html",
|
||||
"sourceSelectors": {
|
||||
"articleBody": ".field.field--name-body",
|
||||
"articleDate": ".views-field-created",
|
||||
"articleLink": ".views-field-title a",
|
||||
"articles": ".view-content > .row.views-row",
|
||||
"articleTitle": ".views-field-title a",
|
||||
"pagination": "ul.pagination > li.pager__item.pager__item--last > a"
|
||||
},
|
||||
"sourceUrl": "https://7sur7.cd",
|
||||
"supportsCategories": true
|
||||
},
|
||||
{
|
||||
"paginationTemplate": "articles.html",
|
||||
"requiresDetails": true,
|
||||
"requiresRateLimit": false,
|
||||
"sourceDate": {
|
||||
"format": "%d.%m.%Y %H:%M"
|
||||
},
|
||||
"sourceId": "mediacongo.net",
|
||||
"sourceKind": "html",
|
||||
"sourceSelectors": {
|
||||
"articleBody": ".article_ttext",
|
||||
"articleCategories": "a.color_link",
|
||||
"articleDate": ".article_other_about",
|
||||
"articleLink": "a:first-child",
|
||||
"articles": ".for_aitems > .article_other_item",
|
||||
"articleTitle": "h1",
|
||||
"pagination": "div.pagination > div > a:last-child"
|
||||
},
|
||||
"sourceUrl": "https://www.mediacongo.net",
|
||||
"supportsCategories": false
|
||||
},
|
||||
{
|
||||
"paginationTemplate": "actualite",
|
||||
"requiresDetails": true,
|
||||
"requiresRateLimit": false,
|
||||
"sourceDate": {
|
||||
"pattern": "/(\\d{1}) (\\d{1,2}) (\\d{2}) (\\d{4}) - (\\d{2}:\\d{2})/",
|
||||
"replacement": "$4-$3-$2 $5"
|
||||
},
|
||||
"sourceId": "actualite.cd",
|
||||
"sourceKind": "html",
|
||||
"sourceSelectors": {
|
||||
"articleBody": ".views-field.views-field-body",
|
||||
"articleCategories": "#actu-cat",
|
||||
"articleDate": "#p-date",
|
||||
"articleLink": "#actu-titre a",
|
||||
"articles": "#views-bootstrap-taxonomy-term-page-2 > div > div",
|
||||
"articleTitle": "h1.page-title"
|
||||
},
|
||||
"sourceUrl": "https://actualite.cd",
|
||||
"supportsCategories": false
|
||||
}
|
||||
],
|
||||
"wordpress": [
|
||||
{
|
||||
"requiresRateLimit": true,
|
||||
"sourceId": "beto.cd",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://beto.cd"
|
||||
},
|
||||
{ "sourceId": "newscd.net", "sourceKind": "wordpress", "sourceUrl": "https://newscd.net" },
|
||||
{
|
||||
"sourceId": "africanewsrdc.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.africanewsrdc.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "angazainstitute.ac.cd",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://angazainstitute.ac.cd"
|
||||
},
|
||||
{ "sourceId": "b-onetv.cd", "sourceKind": "wordpress", "sourceUrl": "https://b-onetv.cd" },
|
||||
{
|
||||
"sourceId": "bukavufm.com",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://bukavufm.com"
|
||||
},
|
||||
{
|
||||
"sourceId": "changement7.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://changement7.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "congoactu.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://congoactu.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "congoindependant.com",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.congoindependant.com"
|
||||
},
|
||||
{
|
||||
"sourceId": "congoquotidien.com",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.congoquotidien.com"
|
||||
},
|
||||
{
|
||||
"sourceId": "cumulard.cd",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.cumulard.cd"
|
||||
},
|
||||
{
|
||||
"sourceId": "environews-rdc.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://environews-rdc.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "freemediardc.info",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.freemediardc.info"
|
||||
},
|
||||
{
|
||||
"sourceId": "geopolismagazine.org",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://geopolismagazine.org"
|
||||
},
|
||||
{
|
||||
"sourceId": "habarirdc.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://habarirdc.net"
|
||||
},
|
||||
{ "sourceId": "infordc.com", "sourceKind": "wordpress", "sourceUrl": "https://infordc.com" },
|
||||
{
|
||||
"sourceId": "kilalopress.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://kilalopress.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "laprosperiteonline.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://laprosperiteonline.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "laprunellerdc.cd",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://laprunellerdc.cd"
|
||||
},
|
||||
{
|
||||
"sourceId": "lesmedias.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://lesmedias.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "lesvolcansnews.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://lesvolcansnews.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "netic-news.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.netic-news.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "objectif-infos.cd",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://objectif-infos.cd"
|
||||
},
|
||||
{
|
||||
"sourceId": "scooprdc.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://scooprdc.net"
|
||||
},
|
||||
{
|
||||
"sourceId": "journaldekinshasa.com",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://www.journaldekinshasa.com"
|
||||
},
|
||||
{
|
||||
"sourceId": "lepotentiel.cd",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://lepotentiel.cd"
|
||||
},
|
||||
{ "sourceId": "acturdc.com", "sourceKind": "wordpress", "sourceUrl": "https://acturdc.com" },
|
||||
{
|
||||
"sourceId": "matininfos.net",
|
||||
"sourceKind": "wordpress",
|
||||
"sourceUrl": "https://matininfos.net"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user