mirror of
https://github.com/hpware/news-analyze.git
synced 2025-06-23 13:04:23 +00:00
Ok? It maybe works?
This commit is contained in:
parent
bb7ab3a088
commit
a0c4d2317e
3 changed files with 9 additions and 9 deletions
135
server/api/home/lt.ts
Normal file
135
server/api/home/lt.ts
Normal file
|
@ -0,0 +1,135 @@
|
|||
// Check /about/scraping_line_today_home.md for more info or https://news.yuanhau.com/datainfo/linetodayjsondata.json
|
||||
interface CacheItem {
|
||||
data: string[];
|
||||
timestamp: number;
|
||||
}
|
||||
const cache: Record<string, CacheItem> = {};
|
||||
const CACHE_DURATION = 1000 * 60 * 60; // 1 Hour
|
||||
|
||||
async function getUUID(orgtype: string) {
|
||||
const type = orgtype.toLowerCase();
|
||||
if (cache[type] && Date.now() - cache[type].timestamp < CACHE_DURATION) {
|
||||
console.log("Serving from cache for type:", type);
|
||||
return {
|
||||
...cache[type].data,
|
||||
cached: true,
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
const buildUrl = `https://today.line.me/_next/data/v1/tw/v3/tab/${type}.json?tabs=${type}`;
|
||||
const req = await fetch(buildUrl, {
|
||||
headers: {
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
Accept: "application/json",
|
||||
"User-Agent":
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||||
},
|
||||
});
|
||||
const res = await req.json();
|
||||
const req2 = res.pageProps.fallback[`getPageData,${type}`].modules;
|
||||
const req3 = [];
|
||||
req2.forEach((key) => {
|
||||
const listings = key.listings;
|
||||
if (Array.isArray(listings)) {
|
||||
listings.forEach((listing) => {
|
||||
if (listing && listing.id) {
|
||||
req3.push(listing.id);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
cache[type] = {
|
||||
data: req3,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
return req3;
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
if (cache[type]) {
|
||||
console.log("Serving expired cache due to error");
|
||||
return cache[type].data;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function filterUUIDs(ids: string[]): string[] {
|
||||
const uuidPattern =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
||||
return ids.filter((id) => uuidPattern.test(id));
|
||||
}
|
||||
|
||||
function filter2(ids: string[]): string[] {
|
||||
const pattern = /news_cat:[a-zA-Z0-9]{24}/g;
|
||||
return ids.filter((id) => pattern.test(id));
|
||||
}
|
||||
|
||||
async function tryToPullDataUUID(uuid: string) {
|
||||
const buildUrl =
|
||||
"https://today.line.me/api/v6/listings/" +
|
||||
uuid +
|
||||
"?country=tw&offset=0&length=27";
|
||||
const req = await fetch(buildUrl);
|
||||
const res = await req.json();
|
||||
return res.items;
|
||||
}
|
||||
|
||||
async function tryToPullDataNUUID(link: string) {
|
||||
const buildUrl =
|
||||
"https://today.line.me/webapi/recommendation/hybrid/listings/" +
|
||||
link +
|
||||
"?country=tw&maxVideoCount=0&offset=0&length=70&optOut=false";
|
||||
const req = await fetch(buildUrl);
|
||||
const res = await req.json();
|
||||
return res;
|
||||
}
|
||||
|
||||
export default defineEventHandler(async (event) => {
|
||||
try {
|
||||
const query = getQuery(event);
|
||||
if (!query.query) {
|
||||
return {
|
||||
error: "NOT_A_QUERY",
|
||||
};
|
||||
}
|
||||
const data = await getUUID(String(query.query));
|
||||
const validUUIDs = filterUUIDs(data || []);
|
||||
const uniqueUUIDs = [...new Set(validUUIDs)];
|
||||
|
||||
const returnData: any[] = await Promise.all(
|
||||
uniqueUUIDs.map(async (key) => {
|
||||
const data = await tryToPullDataUUID(key);
|
||||
return data;
|
||||
}),
|
||||
).then((ans) => ans.flat());
|
||||
const diffFormat = [];
|
||||
data.forEach((key) => {
|
||||
if (validUUIDs.includes(key)) {
|
||||
return;
|
||||
} else {
|
||||
diffFormat.push(key);
|
||||
}
|
||||
});
|
||||
const fillll = filter2(diffFormat);
|
||||
const cconaa: any[] = await Promise.all(
|
||||
fillll.map(async (key) => {
|
||||
const data = await tryToPullDataNUUID(key);
|
||||
return data;
|
||||
}),
|
||||
).then((ans) => ans.flat());
|
||||
return {
|
||||
uuids: uniqueUUIDs,
|
||||
nuuid: fillll,
|
||||
uuidData: returnData,
|
||||
nuuiddata: cconaa,
|
||||
};
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
return {
|
||||
error: "SERVER_SIDE_ERROR",
|
||||
};
|
||||
}
|
||||
});
|
Loading…
Add table
Add a link
Reference in a new issue