43 lines
1.3 KiB
JSON
43 lines
1.3 KiB
JSON
{
|
|
"paths": {
|
|
"root": "%env(BASANGO_CRAWLER_ROOT_PATH)%",
|
|
"data": "%env(BASANGO_CRAWLER_DATA_PATH)%",
|
|
"logs": "%env(BASANGO_CRAWLER_LOGS_PATH)%",
|
|
"config": "%env(BASANGO_CRAWLER_CONFIG_PATH)%"
|
|
},
|
|
"fetch": {
|
|
"client": {
|
|
"timeout": 20,
|
|
"userAgent": "%env(BASANGO_CRAWLER_FETCH_USER_AGENT)%",
|
|
"followRedirects": true,
|
|
"verifySsl": true,
|
|
"rotate": true,
|
|
"maxRetries": "%env(BASANGO_CRAWLER_FETCH_MAX_RETRIES)%",
|
|
"backoffInitial": 1,
|
|
"backoffMultiplier": 2,
|
|
"backoffMax": 30,
|
|
"respectRetryAfter": "%env(BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER)%"
|
|
},
|
|
"crawler": {
|
|
"notify": false,
|
|
"useMultiThreading": false,
|
|
"maxWorkers": 5,
|
|
"direction": "%env(BASANGO_CRAWLER_UPDATE_DIRECTION)%"
|
|
},
|
|
"async": {
|
|
"redisUrl": "%env(BASANGO_CRAWLER_ASYNC_REDIS_URL)%",
|
|
"prefix": "basango:crawler",
|
|
"ttl": {
|
|
"default": 600,
|
|
"result": "%env(BASANGO_CRAWLER_ASYNC_TTL_RESULT)%",
|
|
"failure": "%env(BASANGO_CRAWLER_ASYNC_TTL_FAILURE)%"
|
|
},
|
|
"queues": {
|
|
"listing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_LISTING)%",
|
|
"details": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS)%",
|
|
"processing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING)%"
|
|
}
|
|
}
|
|
}
|
|
}
|