// implementation of just the business/problem specifc logic.
async function getArticlePostParametersDeferred(article) {
const { title, sourceUrl, id, publishedAt } = article;
console.log(`Get post-data of article "${ title }".`);
// nothing here which could be further parallelized.
const markup = await scraper(sourceUrl);
const data = await askGpt(markup, title); // cheating `title` into it for demonstration purpose.
const generatedImageUrl = await generateImg(data?.imageDescription);
const s3ImageUrl = await generateImgUrl(generatedImageUrl, title, id);
// the post-data derieved from a scraped article.
return [
data?.title, data?.abstract, data?.content,
s3ImageUrl, publishedAt, data?.categories,
];
}
async function fetchAndProcessNews(queryString, from) {
// implemented similar to `Promise.allSettled`.
const { promise, resolve, reject } = Promise.withResolvers();
const allSettledData = {
scraping: { resolved: [], rejected: [] },
posting: { resolved: [], rejected: [] },
};
let articleCount = -1;
function settleAllIfPossible() {
const {
scraping: {
resolved: { length: scrapingResolvedCount },
rejected: { length: scrapingRejectedCount },
},
posting: {
resolved: { length: postingResolvedCount },
rejected: { length: postingRejectedCount },
},
} = allSettledData;
if (
(scrapingResolvedCount + scrapingRejectedCount === articleCount) &&
(postingResolvedCount + postingRejectedCount === articleCount)
) {
resolve(allSettledData);
}
}
// play with the queues batch sizes.
// - e.g. batch size of 3 for post requests.
const postTaskQueue = new AsyncTaskExecutionQueue(3); // 4
// - e.g. batch size of 5 for scraping requests.
const scrapeTaskQueue = new AsyncTaskExecutionQueue(5); // 4
postTaskQueue
.addEventListener( rejected , ({ detail: { reasons: postFailureList } }) => {
allSettledData.posting.rejected.push(...postFailureList);
console.log({ postFailureList });
settleAllIfPossible();
});
postTaskQueue
.addEventListener( resolved , ({ detail: { values: postResponseList } }) => {
allSettledData.posting.resolved.push(...postResponseList);
console.log({ postResponseList });
settleAllIfPossible();
});
scrapeTaskQueue
.addEventListener( rejected , ({ detail: { reasons: scrapeFailureList } }) => {
allSettledData.scraping.rejected.push(...scrapeFailureList);
console.log({ scrapeFailureList });
settleAllIfPossible();
});
scrapeTaskQueue
.addEventListener( resolved , ({ detail: { values: postParamsList } }) => {
postTaskQueue
.enqueue(
postParamsList
.map(postParams => ({
createAsynTask: createPost,
params: postParams,
}))
);
allSettledData.scraping.resolved.push(...postParamsList);
// console.log({ postParamsList });
settleAllIfPossible();
});
try {
const { articles } = await searchApi.getNews({ queryString, from, size: 1 });
if (articles) {
articleCount = articles.length;
if (articleCount > 0) {
scrapeTaskQueue
.enqueue(
[...articles]
.map(article => ({
createAsynTask: getArticlePostParametersDeferred,
params: [article],
}))
);
} else {
reject( No articles found. );
}
} else {
reject( Internal error. );
}
} catch (reason) {
reject(reason);
}
return promise;
}
(async () => {
console.log( ... start fetching and processing ... );
const timestamp = Date.now();
try {
const allSettledData = await fetchAndProcessNews();
console.log(`... total fetching and processing time ... ${ Date.now() - timestamp } msec ...`);
console.log({ allSettledData });
} catch (reason) {
console.log(`... total fetching and processing time ... ${ Date.now() - timestamp } msec ...`);
console.error({ reason });
}
})();
.as-console-wrapper { min-height: 100%!important; top: 0; }
<script>
// helper/utility funtionality.
function isFunction(value) {
return (
(typeof value === function ) &&
(typeof value.call === function ) &&
(typeof value.apply === function )
);
}
function isAsynFunction(value) {
return (/[objects+AsyncFunction]/).test(
Object.prototype.toString.call(value)
);
}
function throttle(proceed, threshold = 200, target) {
let timeoutId = null;
let referenceTime = 0;
return (isAsynFunction(proceed) && async function throttled(...args) {
const currentTime = Date.now();
if (currentTime - referenceTime >= threshold) {
clearTimeout(timeoutId);
referenceTime = currentTime;
const trigger = proceed.bind((target ?? this), ...args);
timeoutId = setTimeout((() => {
referenceTime = 0;
trigger();
}), threshold);
trigger();
}
}) || (isFunction(proceed) && function throttled(...args) {
const currentTime = Date.now();
if (currentTime - referenceTime >= threshold) {
clearTimeout(timeoutId);
referenceTime = currentTime;
const trigger = proceed.bind((target ?? this), ...args);
timeoutId = setTimeout((() => {
referenceTime = 0;
trigger();
}), threshold);
trigger();
}
}) || proceed;
}
function getValuesAndReasons(settledItems) {
return settledItems
.reduce(({ values, reasons }, { value, reason = null }) => {
if (reason !== null) {
reasons.push(reason);
} else {
values.push(value);
}
return { values, reasons };
}, { values: [], reasons: [] });
}
</script>
<script>
// abstraction of an "Async Task Execution Queue".
async function* createAsyncTaskBatches(taskComponentsList, batchSize) {
while (taskComponentsList.length >= 1) {
// - try to parallelize the execution of asynchronous tasks by creating
// `Promise.allSettled` based arrays of deferred task execution values.
yield await Promise.allSettled(
taskComponentsList
.splice(0, batchSize)
.map(({ createAsynTask, params }) => createAsynTask(...params))
);
}
}
async function createExecuteAndNotifyAsyncTasks() {
const { queue, taskComponentsList, batchSize } = this;
const deferredResultsPool =
createAsyncTaskBatches(taskComponentsList, batchSize);
for await (const settledItems of deferredResultsPool) {
// console.log({ settledItems });
queue
.dispatchEvent(
new CustomEvent( settled , { detail: { results: settledItems } })
);
const { values, reasons } = getValuesAndReasons(settledItems);
if (values.length > 0) {
queue
.dispatchEvent(
new CustomEvent( resolved , { detail: { values } })
);
}
if (reasons.length > 0) {
queue
.dispatchEvent(
new CustomEvent( rejected , { detail: { reasons } })
);
}
}
}
class AsyncTaskExecutionQueue extends EventTarget {
#createExecuteAndNotify;
#taskComponentsList = [];
constructor(batchSize = 4, throttleThreshold = 0) {
super();
this.#createExecuteAndNotify = throttle(
createExecuteAndNotifyAsyncTasks.bind({
queue: this, taskComponentsList: this.#taskComponentsList, batchSize,
}),
throttleThreshold,
);
}
enqueue(...args) {
const taskComponents = args
.flat()
.filter(({ createAsynTask, params }) =>
isFunction(createAsynTask) && Array.isArray(params)
);
if (taskComponents.length > 0) {
this.#taskComponentsList.push(...taskComponents);
this.#createExecuteAndNotify();
}
}
}
</script>
<script>
// mocking an entire api environment.
const scraper = async (url) =>
new Promise(resolve =>
setTimeout(resolve, (Math.random() * 1500), <markup/> )
);
const askGpt = async (markup, mockedTitle) =>
new Promise(resolve =>
setTimeout(resolve, (Math.random() * 1500), {
imageDescription: foo bar ,
title: mockedTitle,
abstract: TLDR ,
content: Lorem ipsum dolor sit amet ,
categories: [ asynchronous , performance , test ],
})
);
const generateImg = async (description) => {
const name = `${ description.split(/s+/).join( - ).replace(/[^w-]/g, ) }.jpg`;
return new Promise(resolve => setTimeout(resolve, (Math.random() * 500), name));
};
const generateImgUrl = async (name, title, id) => {
title = title.split(/s+/).join( - ).replace(/[^w-]/g, );
return new Promise(resolve =>
setTimeout(resolve, (Math.random() * 500), `s3/bucket/images/${ id }/${ title }/${ name }`)
);
};
const createPost = async (...args) => {
console.log(`Posting scraped article data from arguments ... ["${ args.join( ", " ) }"]`);
return new Promise(resolve =>
setTimeout(resolve, (Math.random() * 1500), { title: args[0] })
);
};
const searchApi = {
getNews: async (query) =>
new Promise(resolve =>
setTimeout(resolve, (Math.random() * 1500), {
//articles: [],
articles: [{
title: 01) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 02) Dolor sit amet ,
sourceUrl: biz/buzz/booz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 03) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 04) Dolor sit amet ,
sourceUrl: biz/buzz/booz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 05) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 06) Dolor sit amet ,
sourceUrl: biz/buzz/booz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 07) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 08) Dolor sit amet ,
sourceUrl: biz/buzz/booz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 09) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 10) Dolor sit amet ,
sourceUrl: biz/buzz/booz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 11) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 12) Dolor sit amet ,
sourceUrl: biz/buzz/booz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}, {
title: 13) Lorem Ipsum ,
sourceUrl: foo/bar/baz ,
id: crypto?.randomUUID?.(),
publishedAt: new Date(Date.now() - (Math.random() * 14*24*3_600_000)).toUTCString(),
}],
})
),
};
</script>