v0.2/multi-origin-website #72
|
@ -17,3 +17,8 @@ export interface VideoData {
|
|||
hiResUrl?: string,
|
||||
hlsUrl?: string
|
||||
}
|
||||
|
||||
export interface VideoAgent {
|
||||
getGallery(params?: FetchParams): Promise<GalleryData[]>
|
||||
getVideo(id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]>
|
||||
}
|
|
@ -2,11 +2,13 @@ import { FetchParams, GalleryData, VideoData } from "@/meta/data";
|
|||
|
||||
import { Platforms } from "@/meta/settings";
|
||||
|
||||
import { fetchXVideosGalleryData } from "./scrape/xvideos/gallery";
|
||||
import { fetchXvideosVideoData } from "./scrape/xvideos/video";
|
||||
import { XVideosAgent } from "./scrape/xvideos/agent";
|
||||
import { XNXXAgent } from "./scrape/xnxx/agent";
|
||||
|
||||
import { fetchXNXXGalleryData } from "./scrape/xnxx/gallery";
|
||||
import { XNXX_BASE_URL } from "@/constants/urls";
|
||||
const AgentMapper = {
|
||||
[Platforms.xvideos]: XVideosAgent,
|
||||
[Platforms.xnxx]: XNXXAgent
|
||||
}
|
||||
|
||||
export class VideoAgent {
|
||||
platform: Platforms;
|
||||
|
@ -16,25 +18,10 @@ export class VideoAgent {
|
|||
}
|
||||
|
||||
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
||||
switch (this.platform) {
|
||||
case Platforms.xvideos:
|
||||
return await fetchXVideosGalleryData(params)
|
||||
case Platforms.xnxx:
|
||||
return await fetchXNXXGalleryData(params)
|
||||
default:
|
||||
return []
|
||||
}
|
||||
return await new AgentMapper[this.platform]().getGallery(params)
|
||||
}
|
||||
|
||||
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||
switch (this.platform) {
|
||||
case Platforms.xvideos:
|
||||
return await fetchXvideosVideoData(id, params)
|
||||
case Platforms.xnxx:
|
||||
return await fetchXvideosVideoData(id, { ...params, baseUrl: XNXX_BASE_URL })
|
||||
default:
|
||||
return [{ lowResUrl: '' }, []]
|
||||
}
|
||||
return await new AgentMapper[this.platform]().getVideo(id, params)
|
||||
}
|
||||
|
||||
}
|
|
@ -1,5 +1,5 @@
|
|||
import { XVIDEOS_BASE_URL } from "@/constants/urls";
|
||||
import { removeHttpS } from "../string";
|
||||
import { removeHttpS } from "@/utils/string";
|
||||
|
||||
const getRandomUserAgent = (): string => {
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import { GalleryData } from "@/meta/data";
|
||||
import { Platforms } from "@/meta/settings";
|
||||
|
||||
export const findVideoUrlInsideTagStringByFunctionNameAndExtension = (
|
||||
tagBlock: string, functionName: string, extension: string): string | null => {
|
||||
const start = tagBlock.indexOf(`html5player.${functionName}('`) + `html5player.${functionName}('`.length;
|
||||
const end = tagBlock.toString().indexOf("'", start);
|
||||
|
||||
const substr = tagBlock.substring(start, end);
|
||||
|
||||
if (substr.includes(extension)) {
|
||||
return substr
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export const findRelatedVideos = (tagBlock: string, platform: Platforms): GalleryData[] | null => {
|
||||
if (!(tagBlock.includes('video_related=['))) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Trova l'inizio e la fine dell'array di oggetti nell'input
|
||||
const start = tagBlock.indexOf('[{');
|
||||
const end = tagBlock.lastIndexOf('}]') + 2;
|
||||
|
||||
// Estrai la sottostringa contenente l'array di oggetti
|
||||
const jsonString = tagBlock.substring(start, end);
|
||||
|
||||
// Parsea la stringa JSON in un array di oggetti
|
||||
const videoRelatedArray = JSON.parse(jsonString);
|
||||
|
||||
// Mappa ogni oggetto nell'array per rinominare le chiavi
|
||||
//@ts-ignore
|
||||
const parsedArray = videoRelatedArray.map(obj => ({
|
||||
//@ts-ignore
|
||||
videoUrl: obj.u,
|
||||
imgUrl: obj.i,
|
||||
text: obj.tf,
|
||||
platform
|
||||
}));
|
||||
|
||||
return parsedArray;
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
|
||||
import { fetchXNXXGalleryData, } from "./gallery";
|
||||
import { fetchXNXXVideoData } from "./video";
|
||||
|
||||
export class XNXXAgent implements VideoAgent {
|
||||
|
||||
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
||||
return await fetchXNXXGalleryData(params)
|
||||
}
|
||||
|
||||
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||
return await fetchXNXXVideoData(id, params)
|
||||
}
|
||||
|
||||
}
|
|
@ -3,7 +3,7 @@ import axios, { AxiosError } from 'axios';
|
|||
|
||||
import * as cheerio from "cheerio";
|
||||
|
||||
import { getHeaders } from '../headers';
|
||||
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||
import { getXNXXQueryUrl } from './url';
|
||||
|
||||
import { Platforms } from '@/meta/settings';
|
||||
|
|
|
@ -0,0 +1,71 @@
|
|||
import { XNXX_BASE_URL } from '@/constants/urls';
|
||||
import { FetchParams, GalleryData, VideoData } from '@/meta/data';
|
||||
|
||||
import axios, { AxiosError } from 'axios';
|
||||
|
||||
import * as cheerio from "cheerio";
|
||||
|
||||
import { Platforms } from '@/meta/settings';
|
||||
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
|
||||
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||
|
||||
export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||
|
||||
let data: VideoData = {
|
||||
lowResUrl: ''
|
||||
}
|
||||
|
||||
let related: GalleryData[] = [];
|
||||
|
||||
const host = XNXX_BASE_URL
|
||||
|
||||
const reqHeaders = getHeaders(host)
|
||||
|
||||
const queryUrl = `${host}${videoId}`
|
||||
|
||||
await axios.get(queryUrl, reqHeaders)
|
||||
|
||||
.then(response => {
|
||||
|
||||
const html = response.data;
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const scriptTags = $("script");
|
||||
|
||||
// populate video data object
|
||||
scriptTags.map((idx, elem) => {
|
||||
|
||||
const lowResUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoUrlLow', '.mp4')
|
||||
const hiResUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoUrlHigh', '.mp4')
|
||||
const hlsUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoHLS', '.m3u8')
|
||||
|
||||
if (lowResUrl) {
|
||||
data.lowResUrl = lowResUrl;
|
||||
}
|
||||
|
||||
if (hiResUrl) {
|
||||
data.hiResUrl = hiResUrl
|
||||
}
|
||||
|
||||
if (hlsUrl) {
|
||||
data.hlsUrl = hlsUrl
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
// populate related gallery
|
||||
scriptTags.map((idx, elem) => {
|
||||
const relatedVideos = findRelatedVideos($(elem).toString(), Platforms.xnxx)
|
||||
|
||||
if (relatedVideos) {
|
||||
related = relatedVideos
|
||||
}
|
||||
})
|
||||
|
||||
}).catch((error: AxiosError) => {
|
||||
// handle errors
|
||||
});
|
||||
|
||||
return [data, related];
|
||||
}
|
|
@ -0,0 +1,15 @@
|
|||
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
|
||||
import { fetchXVideosGalleryData } from "./gallery";
|
||||
import { fetchXvideosVideoData } from "./video";
|
||||
|
||||
export class XVideosAgent implements VideoAgent {
|
||||
|
||||
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
||||
return await fetchXVideosGalleryData(params)
|
||||
}
|
||||
|
||||
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||
return await fetchXvideosVideoData(id, params)
|
||||
}
|
||||
|
||||
}
|
|
@ -2,7 +2,7 @@ import { FetchParams, GalleryData } from '@/meta/data';
|
|||
import axios, { AxiosError } from 'axios';
|
||||
|
||||
import * as cheerio from "cheerio";
|
||||
import { getHeaders } from '../headers';
|
||||
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||
import { getXVideosQueryUrl } from './url';
|
||||
import { Platforms } from '@/meta/settings';
|
||||
|
||||
|
|
|
@ -4,8 +4,10 @@ import { FetchParams, GalleryData, VideoData } from '@/meta/data';
|
|||
import axios, { AxiosError } from 'axios';
|
||||
|
||||
import * as cheerio from "cheerio";
|
||||
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '../../string';
|
||||
import { getHeaders } from '../headers';
|
||||
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||
import { Platforms } from '@/meta/settings';
|
||||
|
||||
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
|
||||
|
||||
export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||
|
||||
|
@ -15,9 +17,11 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
|
|||
|
||||
let related: GalleryData[] = [];
|
||||
|
||||
const reqHeaders = getHeaders()
|
||||
const host = XVIDEOS_BASE_URL
|
||||
|
||||
const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${videoId}`
|
||||
const reqHeaders = getHeaders(host)
|
||||
|
||||
const queryUrl = `${host}${videoId}`
|
||||
|
||||
await axios.get(queryUrl, reqHeaders)
|
||||
|
||||
|
@ -25,7 +29,6 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
|
|||
|
||||
const html = response.data;
|
||||
|
||||
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
const scriptTags = $("script");
|
||||
|
@ -53,7 +56,7 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
|
|||
|
||||
// populate related gallery
|
||||
scriptTags.map((idx, elem) => {
|
||||
const relatedVideos = findRelatedVideos($(elem).toString())
|
||||
const relatedVideos = findRelatedVideos($(elem).toString(), Platforms.xvideos)
|
||||
|
||||
if (relatedVideos) {
|
||||
related = relatedVideos
|
||||
|
|
|
@ -1,46 +1,3 @@
|
|||
import { GalleryData } from "@/meta/data";
|
||||
|
||||
export const findVideoUrlInsideTagStringByFunctionNameAndExtension = (
|
||||
tagBlock: string, functionName: string, extension: string): string | null => {
|
||||
const start = tagBlock.indexOf(`html5player.${functionName}('`) + `html5player.${functionName}('`.length;
|
||||
const end = tagBlock.toString().indexOf("'", start);
|
||||
|
||||
const substr = tagBlock.substring(start, end);
|
||||
|
||||
if (substr.includes(extension)) {
|
||||
return substr
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
|
||||
export const findRelatedVideos = (tagBlock: string): GalleryData[] | null => {
|
||||
if (!(tagBlock.includes('video_related=['))) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Trova l'inizio e la fine dell'array di oggetti nell'input
|
||||
const start = tagBlock.indexOf('[{');
|
||||
const end = tagBlock.lastIndexOf('}]') + 2;
|
||||
|
||||
// Estrai la sottostringa contenente l'array di oggetti
|
||||
const jsonString = tagBlock.substring(start, end);
|
||||
|
||||
// Parsea la stringa JSON in un array di oggetti
|
||||
const videoRelatedArray = JSON.parse(jsonString);
|
||||
|
||||
// Mappa ogni oggetto nell'array per rinominare le chiavi
|
||||
//@ts-ignore
|
||||
const parsedArray = videoRelatedArray.map(obj => ({
|
||||
//@ts-ignore
|
||||
videoUrl: obj.u,
|
||||
imgUrl: obj.i,
|
||||
text: obj.tf
|
||||
}));
|
||||
|
||||
return parsedArray;
|
||||
}
|
||||
|
||||
export const removeHttpS = (url: string): string => {
|
||||
if (url.startsWith("http://")) {
|
||||
return url.slice(7);
|
||||
|
|
Loading…
Reference in New Issue