add random user-agent and more request headers

src/utils/scrape/gallery.ts

@@ -1,8 +1,9 @@
 import { XVIDEOS_BASE_URL } from '@/constants/urls';
-import { GalleryData, VideoData } from '@/meta/data';
+import { GalleryData } from '@/meta/data';
 import axios, { AxiosError } from 'axios';
 
 import * as cheerio from "cheerio";
+import { getHeaders } from './headers';
 
 interface FetchParams {
     baseUrl?: string
@@ -13,11 +14,7 @@ export const fetchGalleryData = async (params?: FetchParams): Promise<GalleryDat
 
     let data: GalleryData[] = [];
 
-    const reqHeaders = {
+    const reqHeaders = getHeaders()
-        headers: {
-            "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5397.215 Safari/537.36'
-        },
-    };
 
     const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${params && params.query ? '/?k=' + params.query : ''}`
 

src/utils/scrape/headers.ts

@@ -0,0 +1,38 @@
+import { XVIDEOS_BASE_URL } from "@/constants/urls";
+import { removeHttpS } from "../string";
+
+const getRandomUserAgent = (): string => {
+
+    const userAgents: string[] = [
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5397.215 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 Edg/123.0.2420.81',
+        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.4; rv:124.0) Gecko/20100101 Firefox/124.0',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4.1 Safari/605.1.15',
+        'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/109.0.0.0',
+        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',
+        'Mozilla/5.0 (X11; Linux i686; rv:124.0) Gecko/20100101 Firefox/124.0'
+    ];
+
+    const rand = Math.floor(Math.random() * userAgents.length);
+
+    return userAgents[rand]
+}
+
+export const getHeaders = (host:string = XVIDEOS_BASE_URL) => {
+    return {
+        headers: {
+            "User-Agent": getRandomUserAgent(),
+            "Accept-Language": "en-gb, en, en-US, it",
+            "Accept-Encoding": "gzip, deflate, br",
+            "Connection": "keep-alive",
+            "Sec-Fetch-Dest": "document",
+            "Sec-Fetch-Mode": "navigate",
+            "Sec-Fetch-Site": "none",
+            "Host": removeHttpS(host)
+        },
+    }
+};

src/utils/scrape/video.ts

@@ -5,6 +5,7 @@ import axios, { AxiosError } from 'axios';
 
 import * as cheerio from "cheerio";
 import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '../string';
+import { getHeaders } from './headers';
 
 interface FetchParams {
     baseUrl?: string
@@ -19,11 +20,7 @@ export const fetchVideoData = async (videoId: string, params?: FetchParams): Pro
 
     let related: GalleryData[] = [];
 
-    const reqHeaders = {
+    const reqHeaders = getHeaders()
-        headers: {
-            "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_14) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.5397.215 Safari/537.36'
-        },
-    };
 
     const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${videoId}`
 

src/utils/string.ts

@@ -39,4 +39,13 @@ export const findRelatedVideos = (tagBlock: string): GalleryData[]|null => {
     }));
     
     return parsedArray;
-}
+}
+
+export const removeHttpS = (url: string): string => {
+    if (url.startsWith("http://")) {
+      return url.slice(7);
+    } else if (url.startsWith("https://")) {
+      return url.slice(8);
+    }
+    return url;
+  };