lamacchinadesiderante/proxyraye-nextjs
lamacchinadesiderante
/
proxyraye-nextjs
0
0
Fork 0
Code Issues 20 Pull Requests Packages Projects 6 Releases 7 Wiki Activity

add random user-agent and more request headers

This commit is contained in:
lamacchinadesiderante 2024-04-28 13:17:14 +02:00
parent f2efe33dba
commit c98ad299cc
4 changed files with 53 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/utils/scrape/gallery.ts
View File

@ -1,8 +1,9 @@
import { XVIDEOS_BASE_URL } from '@/constants/urls';
import { GalleryData, VideoData } from '@/meta/data';
import { GalleryData } from '@/meta/data';
import axios, { AxiosError } from 'axios';
import * as cheerio from "cheerio";
import { getHeaders } from './headers';
interface FetchParams {
baseUrl?: string
@ -13,11 +14,7 @@ export const fetchGalleryData = async (params?: FetchParams): Promise<GalleryDat
let data: GalleryData[] = [];
const reqHeaders = {
headers: {
"User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5397.215 Safari/537.36'
},
};
const reqHeaders = getHeaders()
const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${params && params.query ? '/?k=' + params.query : ''}`

38
src/utils/scrape/headers.ts Normal file
View File

@ -0,0 +1,38 @@
import { XVIDEOS_BASE_URL } from "@/constants/urls";
import { removeHttpS } from "../string";
const getRandomUserAgent = (): string => {
const userAgents: string[] = [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5397.215 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.2420.81',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.4; rv:124.0) Gecko/20100101 Firefox/124.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
'Mozilla/5.0 (X11; Linux i686; rv:124.0) Gecko/20100101 Firefox/124.0'
];
const rand = Math.floor(Math.random() * userAgents.length);
return userAgents[rand]
}
export const getHeaders = (host:string = XVIDEOS_BASE_URL) => {
return {
headers: {
"User-Agent": getRandomUserAgent(),
"Accept-Language": "en-gb, en, en-US, it",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Host": removeHttpS(host)
},
}
};

7
src/utils/scrape/video.ts
View File

@ -5,6 +5,7 @@ import axios, { AxiosError } from 'axios';
import * as cheerio from "cheerio";
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '../string';
import { getHeaders } from './headers';
interface FetchParams {
baseUrl?: string
@ -19,11 +20,7 @@ export const fetchVideoData = async (videoId: string, params?: FetchParams): Pro
let related: GalleryData[] = [];
const reqHeaders = {
headers: {
"User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5397.215 Safari/537.36'
},
};
const reqHeaders = getHeaders()
const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${videoId}`

11
src/utils/string.ts
View File

@ -39,4 +39,13 @@ export const findRelatedVideos = (tagBlock: string): GalleryData[]|null => {
}));
return parsedArray;
}
}
export const removeHttpS = (url: string): string => {
if (url.startsWith("http://")) {
return url.slice(7);
} else if (url.startsWith("https://")) {
return url.slice(8);
}
return url;
};