Skip to content

Commit

Permalink
Development (#37)
Browse files Browse the repository at this point in the history 
* abortsignal

* ip address

* timeouts
  • Loading branch information
@tsaridas
tsaridas authored Jan 28, 2024
1 parent 67c11b8 commit a00d14a
Show file tree
Hide file tree
Showing 4 changed files with 45 additions and 22 deletions.
1 change: 0 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ The below options can be set as an evironment variable.
| `JACKETT_APIKEYS` | '' | `sdfsadfs,sdfsadfsa` | API keys from jackett servers comma delimited. |
| `ADDON_NAME` | 'Jackett' | `MyJacketAddon` | The name of the addon that will show in stremio. |
| `JACKETT_RTIMEOUT` | 8000 | `20000` | Jackett http read timeout in millisecond. Don't set these higher than the RESPONSE_TIMEOUT. |
| `JACKETT_OTIMEOUT` | 3000 | `20000` | Jackett http open timeout in millisecond. This is how long it takes to open a tcp connection to jackett. Increase if your jackett server is far away from the addon.|
| `DONT_PARSE_TORRENT_FILES` | false | `true` | Parsing torrent files ( not magnets) takes time and is slow. This is disabled by default. **If enabled you will see less results depending on your indexer**. |
| `DOWNLOAD_TORRENT_QUEUE` | 10 | `100` | Because external http downloads go through Jackett doing many downloads at the same time might cause some DDOS so I setup a queue for this. |
| `RESPONSE_TIMEOUT` | 8000 | `12000` | This will timeout any queries to jackett after this given value in millisecond. The higher the most result you will get from slow indexers. |
Expand Down
4 changes: 1 addition & 3 deletions src/config.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@ const defaultConfig = {

"apiKeys": process.env.JACKETT_APIKEYS || process.env.JACKETT_APIKEY || "", // JACKETT_APIKEY is for backwards compatibility

"readTimeout": parseInt(process.env.JACKETT_RTIMEOUT) || 8000, // don't set this lower than response timeout

"openTimeout": parseInt(process.env.JACKETT_OTIMEOUT) || 3000, // this is how long it takes to open a tcp connection to jackett. increase if your jackett server is far away from the addon.
"readTimeout": parseInt(process.env.JACKETT_RTIMEOUT) || 8000,

"indexerFilters": process.env.INDEXER_FILTERS || "status:healthy,test:passed" // instead of `all`.
},
Expand Down
41 changes: 27 additions & 14 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,17 @@ const manifest = {
"catalogs": []
};

addon.get('/manifest.json', (_, res) => {
config.debug && console.log("Sending manifest.");
addon.get('/manifest.json', (req, res) => {
console.log("Sending manifest to .", req.ip);
respond(res, manifest);
});

async function getConemataInfo(streamInfo, signal) {
async function getConemataInfo(streamInfo, abortSignals) {
const url = 'https://v3-cinemeta.strem.io/meta/' + streamInfo.type + '/' + streamInfo.imdbId + '.json';
config.debug && console.log("Cinemata url", url);
const controller = new AbortController();
abortSignals.push(controller)
const signal = controller.signal;

const response = await axios({
method: 'get',
Expand Down Expand Up @@ -210,7 +213,7 @@ function streamFromParsed(tor, parsedTorrent, streamInfo, cb) {
cb(stream);
}

async function addResults(info, streams, source, signal) {
async function addResults(info, streams, source, abortSignals) {

const [url, name] = source.split("||").length === 2 ? source.split("||") : [null, null];
if (!url && !name) {
Expand All @@ -219,6 +222,10 @@ async function addResults(info, streams, source, signal) {
}

try {
const controller = new AbortController();
abortSignals.push(controller);
const signal = controller.signal;

const streamUrl = url + info.type + '/' + info.imdbId + (info.season && info.episode ? ':' + info.season + ':' + info.episode + '.json' : '.json');
config.debug && console.log('Additional source url is :', streamUrl)
const response = await axios.get(streamUrl, {
Expand Down Expand Up @@ -276,8 +283,7 @@ addon.get('/stream/:type/:id.json', async (req, res) => {

let streamInfo = {};
const streams = [];
const controller = new AbortController();
const signal = controller.signal;
const abortSignals = [];

const startTime = Date.now();

Expand All @@ -294,19 +300,19 @@ addon.get('/stream/:type/:id.json', async (req, res) => {


try {
await getConemataInfo(streamInfo, signal);
await getConemataInfo(streamInfo, abortSignals);
} catch (err) {
console.error(err.message);
return respond(res, { streams: [] });
}

if (config.additionalSources) {
config.additionalSources.forEach(source => {
addResults(streamInfo, streams, source, signal);
addResults(streamInfo, streams, source, abortSignals);
});
}

console.log(`Q / imdbiID: ${streamInfo.imdbId} / title: ${streamInfo.name} / type: ${streamInfo.type} / year: ${streamInfo.year}` +
console.log(`Q: ip: ${req.ip} / imdbiID: ${streamInfo.imdbId} / title: ${streamInfo.name} / type: ${streamInfo.type} / year: ${streamInfo.year}` +
(streamInfo.season && streamInfo.episode ? ` / season: ${streamInfo.season} / episode: ${streamInfo.episode}` : '') +
'.');

Expand All @@ -319,11 +325,15 @@ addon.get('/stream/:type/:id.json', async (req, res) => {
if (!requestSent && ((elapsedTime >= config.responseTimeout) || (searchFinished && inProgressCount === 0 && asyncQueue.idle))) {
requestSent = true;
asyncQueue.kill();
controller.abort();
config.debug && console.log("There are " + abortSignals.length + " controllers to abort.");
abortSignals.forEach((controller) => {
controller.abort();
});

clearInterval(intervalId);
const finalData = processTorrentList(streams);
config.debug && console.log("Sliced & Sorted data ", finalData);
console.log("A / imdbiID: " + streamInfo.imdbId + " / Results " + finalData.length + " / Timeout: " + (elapsedTime >= config.responseTimeout) + " / Search Finished: " + searchFinished + " / Queue Idle: " + asyncQueue.idle() + " / Pending Downloads : " + inProgressCount + " / Discarded : " + (streams.length - finalData.length));
console.log(`A ip: ${req.ip} / imdbiID: ${streamInfo.imdbId} / Results ${finalData.length} / Timeout: ${(elapsedTime >= config.responseTimeout)} / Search Finished: ${searchFinished} / Queue Idle: ${asyncQueue.idle()} / Pending Downloads : ${inProgressCount} / Discarded : ${(streams.length - finalData.length)}`);
return respond(res, {
streams: finalData,
"cacheMaxAge": 1440,
Expand Down Expand Up @@ -355,12 +365,15 @@ addon.get('/stream/:type/:id.json', async (req, res) => {
}
inProgressCount++;
try {
const controller = new AbortController();
const signal = controller.signal;
abortSignals.push(controller)
config.debug && console.log("Processing link: ", task.link);
const response = await axios.get(task.link, {
timeout: 5000, // we don't want to overdo it here and neither set something in config. Request should timeout anyway.
timeout: config.responseTimeout, // we don't want to overdo it here and neither set something in config. Request should timeout anyway.
maxRedirects: 0, // Equivalent to 'redirect: 'manual'' in fetch
validateStatus: null,
cancelToken: signal.token,
signal: signal,
responseType: 'arraybuffer', // Specify the response type as 'arraybuffer'
});

Expand Down Expand Up @@ -402,7 +415,7 @@ addon.get('/stream/:type/:id.json', async (req, res) => {
const asyncQueue = async.queue(processLinks, config.downloadTorrentQueue);


jackettApi.search(streamInfo, signal,
jackettApi.search(streamInfo, abortSignals,
(tempResults) => {
if (!requestSent && tempResults && tempResults.length > 0) {
const { magnets, links } = partitionURL(tempResults);
Expand Down
21 changes: 17 additions & 4 deletions src/jackett.js
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,19 @@
const xmlJs = require('xml-js');
const axios = require('axios');
const { AbortController } = require('abort-controller');
const helper = require('./helpers');
const config = require('./config');

const getIndexers = async (host, apiKey) => {
const getIndexers = async (host, apiKey, abortSignals) => {
try {
const controller = new AbortController();
abortSignals.push(controller)
const signal = controller.signal;

const response = await axios.get(host + 'api/v2.0/indexers/' + config.jackett.indexerFilters + '/results/torznab/api?apikey=' + apiKey + '&t=indexers&configured=true', {
timeout: config.jackett.readTimeout, // Equivalent to 'read_timeout' in needle
responseType: 'text',
signal: signal
});

if (!response || !response.data) {
Expand Down Expand Up @@ -37,9 +43,10 @@ const getIndexers = async (host, apiKey) => {
}
};

const search = async (query, signal, cb, end) => {
const search = async (query, abortSignals, cb, end) => {
const hostsAndApiKeys = config.jackett.hosts.split(',').map((host, i) => ({ host, apiKey: config.jackett.apiKeys.split(',')[i] }));
config.debug && console.log("Found " + hostsAndApiKeys.length + " Jacket servers.");

let searchQuery = "";
let countResults = 0;
let countFinished = 0;
Expand Down Expand Up @@ -69,10 +76,13 @@ const search = async (query, signal, cb, end) => {
}

await Promise.all(hostsAndApiKeys.map(async ({ host, apiKey }) => {
const apiIndexersArray = await getIndexers(host, apiKey);
const apiIndexersArray = await getIndexers(host, apiKey, abortSignals);

try {
config.debug && console.log("Found " + apiIndexersArray.length + " indexers for " + host);
if (apiIndexersArray.length == 0) {
return;
}

await Promise.all(apiIndexersArray.map(async (indexer) => {
if (!(indexer && indexer.attributes && indexer.attributes.id)) {
Expand All @@ -86,6 +96,10 @@ const search = async (query, signal, cb, end) => {
searchedIndexers[indexer.attributes.id] = { "host": host, "status": "started" };
}

const controller = new AbortController();
abortSignals.push(controller)
const signal = controller.signal;

const url = host + 'api/v2.0/indexers/' + indexer.attributes.id + '/results/torznab/api?apikey=' + apiKey + searchQuery;
const response = await axios.get(url, {
timeout: config.jackett.readTimeout,
Expand Down Expand Up @@ -185,7 +199,6 @@ const search = async (query, signal, cb, end) => {
}));
} catch (error) {
console.error("Could not process host :", host, error.message);

}
}));
end([]);
Expand Down

0 comments on commit a00d14a

Please sign in to comment.