v0.2/multi-origin-website #72
|
@ -17,3 +17,8 @@ export interface VideoData {
|
||||||
hiResUrl?: string,
|
hiResUrl?: string,
|
||||||
hlsUrl?: string
|
hlsUrl?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface VideoAgent {
|
||||||
|
getGallery(params?: FetchParams): Promise<GalleryData[]>
|
||||||
|
getVideo(id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]>
|
||||||
|
}
|
|
@ -2,11 +2,13 @@ import { FetchParams, GalleryData, VideoData } from "@/meta/data";
|
||||||
|
|
||||||
import { Platforms } from "@/meta/settings";
|
import { Platforms } from "@/meta/settings";
|
||||||
|
|
||||||
import { fetchXVideosGalleryData } from "./scrape/xvideos/gallery";
|
import { XVideosAgent } from "./scrape/xvideos/agent";
|
||||||
import { fetchXvideosVideoData } from "./scrape/xvideos/video";
|
import { XNXXAgent } from "./scrape/xnxx/agent";
|
||||||
|
|
||||||
import { fetchXNXXGalleryData } from "./scrape/xnxx/gallery";
|
const AgentMapper = {
|
||||||
import { XNXX_BASE_URL } from "@/constants/urls";
|
[Platforms.xvideos]: XVideosAgent,
|
||||||
|
[Platforms.xnxx]: XNXXAgent
|
||||||
|
}
|
||||||
|
|
||||||
export class VideoAgent {
|
export class VideoAgent {
|
||||||
platform: Platforms;
|
platform: Platforms;
|
||||||
|
@ -16,25 +18,10 @@ export class VideoAgent {
|
||||||
}
|
}
|
||||||
|
|
||||||
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
||||||
switch (this.platform) {
|
return await new AgentMapper[this.platform]().getGallery(params)
|
||||||
case Platforms.xvideos:
|
|
||||||
return await fetchXVideosGalleryData(params)
|
|
||||||
case Platforms.xnxx:
|
|
||||||
return await fetchXNXXGalleryData(params)
|
|
||||||
default:
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||||
switch (this.platform) {
|
return await new AgentMapper[this.platform]().getVideo(id, params)
|
||||||
case Platforms.xvideos:
|
|
||||||
return await fetchXvideosVideoData(id, params)
|
|
||||||
case Platforms.xnxx:
|
|
||||||
return await fetchXvideosVideoData(id, { ...params, baseUrl: XNXX_BASE_URL })
|
|
||||||
default:
|
|
||||||
return [{ lowResUrl: '' }, []]
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
|
@ -1,5 +1,5 @@
|
||||||
import { XVIDEOS_BASE_URL } from "@/constants/urls";
|
import { XVIDEOS_BASE_URL } from "@/constants/urls";
|
||||||
import { removeHttpS } from "../string";
|
import { removeHttpS } from "@/utils/string";
|
||||||
|
|
||||||
const getRandomUserAgent = (): string => {
|
const getRandomUserAgent = (): string => {
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
import { GalleryData } from "@/meta/data";
|
||||||
|
import { Platforms } from "@/meta/settings";
|
||||||
|
|
||||||
|
export const findVideoUrlInsideTagStringByFunctionNameAndExtension = (
|
||||||
|
tagBlock: string, functionName: string, extension: string): string | null => {
|
||||||
|
const start = tagBlock.indexOf(`html5player.${functionName}('`) + `html5player.${functionName}('`.length;
|
||||||
|
const end = tagBlock.toString().indexOf("'", start);
|
||||||
|
|
||||||
|
const substr = tagBlock.substring(start, end);
|
||||||
|
|
||||||
|
if (substr.includes(extension)) {
|
||||||
|
return substr
|
||||||
|
}
|
||||||
|
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
export const findRelatedVideos = (tagBlock: string, platform: Platforms): GalleryData[] | null => {
|
||||||
|
if (!(tagBlock.includes('video_related=['))) {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
// Trova l'inizio e la fine dell'array di oggetti nell'input
|
||||||
|
const start = tagBlock.indexOf('[{');
|
||||||
|
const end = tagBlock.lastIndexOf('}]') + 2;
|
||||||
|
|
||||||
|
// Estrai la sottostringa contenente l'array di oggetti
|
||||||
|
const jsonString = tagBlock.substring(start, end);
|
||||||
|
|
||||||
|
// Parsea la stringa JSON in un array di oggetti
|
||||||
|
const videoRelatedArray = JSON.parse(jsonString);
|
||||||
|
|
||||||
|
// Mappa ogni oggetto nell'array per rinominare le chiavi
|
||||||
|
//@ts-ignore
|
||||||
|
const parsedArray = videoRelatedArray.map(obj => ({
|
||||||
|
//@ts-ignore
|
||||||
|
videoUrl: obj.u,
|
||||||
|
imgUrl: obj.i,
|
||||||
|
text: obj.tf,
|
||||||
|
platform
|
||||||
|
}));
|
||||||
|
|
||||||
|
return parsedArray;
|
||||||
|
}
|
|
@ -0,0 +1,15 @@
|
||||||
|
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
|
||||||
|
import { fetchXNXXGalleryData, } from "./gallery";
|
||||||
|
import { fetchXNXXVideoData } from "./video";
|
||||||
|
|
||||||
|
export class XNXXAgent implements VideoAgent {
|
||||||
|
|
||||||
|
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
||||||
|
return await fetchXNXXGalleryData(params)
|
||||||
|
}
|
||||||
|
|
||||||
|
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||||
|
return await fetchXNXXVideoData(id, params)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -3,7 +3,7 @@ import axios, { AxiosError } from 'axios';
|
||||||
|
|
||||||
import * as cheerio from "cheerio";
|
import * as cheerio from "cheerio";
|
||||||
|
|
||||||
import { getHeaders } from '../headers';
|
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||||
import { getXNXXQueryUrl } from './url';
|
import { getXNXXQueryUrl } from './url';
|
||||||
|
|
||||||
import { Platforms } from '@/meta/settings';
|
import { Platforms } from '@/meta/settings';
|
||||||
|
|
|
@ -0,0 +1,71 @@
|
||||||
|
import { XNXX_BASE_URL } from '@/constants/urls';
|
||||||
|
import { FetchParams, GalleryData, VideoData } from '@/meta/data';
|
||||||
|
|
||||||
|
import axios, { AxiosError } from 'axios';
|
||||||
|
|
||||||
|
import * as cheerio from "cheerio";
|
||||||
|
|
||||||
|
import { Platforms } from '@/meta/settings';
|
||||||
|
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
|
||||||
|
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||||
|
|
||||||
|
export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||||
|
|
||||||
|
let data: VideoData = {
|
||||||
|
lowResUrl: ''
|
||||||
|
}
|
||||||
|
|
||||||
|
let related: GalleryData[] = [];
|
||||||
|
|
||||||
|
const host = XNXX_BASE_URL
|
||||||
|
|
||||||
|
const reqHeaders = getHeaders(host)
|
||||||
|
|
||||||
|
const queryUrl = `${host}${videoId}`
|
||||||
|
|
||||||
|
await axios.get(queryUrl, reqHeaders)
|
||||||
|
|
||||||
|
.then(response => {
|
||||||
|
|
||||||
|
const html = response.data;
|
||||||
|
|
||||||
|
const $ = cheerio.load(html);
|
||||||
|
|
||||||
|
const scriptTags = $("script");
|
||||||
|
|
||||||
|
// populate video data object
|
||||||
|
scriptTags.map((idx, elem) => {
|
||||||
|
|
||||||
|
const lowResUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoUrlLow', '.mp4')
|
||||||
|
const hiResUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoUrlHigh', '.mp4')
|
||||||
|
const hlsUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoHLS', '.m3u8')
|
||||||
|
|
||||||
|
if (lowResUrl) {
|
||||||
|
data.lowResUrl = lowResUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hiResUrl) {
|
||||||
|
data.hiResUrl = hiResUrl
|
||||||
|
}
|
||||||
|
|
||||||
|
if (hlsUrl) {
|
||||||
|
data.hlsUrl = hlsUrl
|
||||||
|
}
|
||||||
|
|
||||||
|
})
|
||||||
|
|
||||||
|
// populate related gallery
|
||||||
|
scriptTags.map((idx, elem) => {
|
||||||
|
const relatedVideos = findRelatedVideos($(elem).toString(), Platforms.xnxx)
|
||||||
|
|
||||||
|
if (relatedVideos) {
|
||||||
|
related = relatedVideos
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
}).catch((error: AxiosError) => {
|
||||||
|
// handle errors
|
||||||
|
});
|
||||||
|
|
||||||
|
return [data, related];
|
||||||
|
}
|
|
@ -0,0 +1,15 @@
|
||||||
|
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
|
||||||
|
import { fetchXVideosGalleryData } from "./gallery";
|
||||||
|
import { fetchXvideosVideoData } from "./video";
|
||||||
|
|
||||||
|
export class XVideosAgent implements VideoAgent {
|
||||||
|
|
||||||
|
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
|
||||||
|
return await fetchXVideosGalleryData(params)
|
||||||
|
}
|
||||||
|
|
||||||
|
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||||
|
return await fetchXvideosVideoData(id, params)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -2,7 +2,7 @@ import { FetchParams, GalleryData } from '@/meta/data';
|
||||||
import axios, { AxiosError } from 'axios';
|
import axios, { AxiosError } from 'axios';
|
||||||
|
|
||||||
import * as cheerio from "cheerio";
|
import * as cheerio from "cheerio";
|
||||||
import { getHeaders } from '../headers';
|
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||||
import { getXVideosQueryUrl } from './url';
|
import { getXVideosQueryUrl } from './url';
|
||||||
import { Platforms } from '@/meta/settings';
|
import { Platforms } from '@/meta/settings';
|
||||||
|
|
||||||
|
|
|
@ -4,8 +4,10 @@ import { FetchParams, GalleryData, VideoData } from '@/meta/data';
|
||||||
import axios, { AxiosError } from 'axios';
|
import axios, { AxiosError } from 'axios';
|
||||||
|
|
||||||
import * as cheerio from "cheerio";
|
import * as cheerio from "cheerio";
|
||||||
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '../../string';
|
import { getHeaders } from '@/utils/scrape/common/headers';
|
||||||
import { getHeaders } from '../headers';
|
import { Platforms } from '@/meta/settings';
|
||||||
|
|
||||||
|
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
|
||||||
|
|
||||||
export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
|
||||||
|
|
||||||
|
@ -15,9 +17,11 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
|
||||||
|
|
||||||
let related: GalleryData[] = [];
|
let related: GalleryData[] = [];
|
||||||
|
|
||||||
const reqHeaders = getHeaders()
|
const host = XVIDEOS_BASE_URL
|
||||||
|
|
||||||
const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${videoId}`
|
const reqHeaders = getHeaders(host)
|
||||||
|
|
||||||
|
const queryUrl = `${host}${videoId}`
|
||||||
|
|
||||||
await axios.get(queryUrl, reqHeaders)
|
await axios.get(queryUrl, reqHeaders)
|
||||||
|
|
||||||
|
@ -25,7 +29,6 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
|
||||||
|
|
||||||
const html = response.data;
|
const html = response.data;
|
||||||
|
|
||||||
|
|
||||||
const $ = cheerio.load(html);
|
const $ = cheerio.load(html);
|
||||||
|
|
||||||
const scriptTags = $("script");
|
const scriptTags = $("script");
|
||||||
|
@ -53,7 +56,7 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
|
||||||
|
|
||||||
// populate related gallery
|
// populate related gallery
|
||||||
scriptTags.map((idx, elem) => {
|
scriptTags.map((idx, elem) => {
|
||||||
const relatedVideos = findRelatedVideos($(elem).toString())
|
const relatedVideos = findRelatedVideos($(elem).toString(), Platforms.xvideos)
|
||||||
|
|
||||||
if (relatedVideos) {
|
if (relatedVideos) {
|
||||||
related = relatedVideos
|
related = relatedVideos
|
||||||
|
|
|
@ -1,46 +1,3 @@
|
||||||
import { GalleryData } from "@/meta/data";
|
|
||||||
|
|
||||||
export const findVideoUrlInsideTagStringByFunctionNameAndExtension = (
|
|
||||||
tagBlock: string, functionName: string, extension: string): string | null => {
|
|
||||||
const start = tagBlock.indexOf(`html5player.${functionName}('`) + `html5player.${functionName}('`.length;
|
|
||||||
const end = tagBlock.toString().indexOf("'", start);
|
|
||||||
|
|
||||||
const substr = tagBlock.substring(start, end);
|
|
||||||
|
|
||||||
if (substr.includes(extension)) {
|
|
||||||
return substr
|
|
||||||
}
|
|
||||||
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
export const findRelatedVideos = (tagBlock: string): GalleryData[] | null => {
|
|
||||||
if (!(tagBlock.includes('video_related=['))) {
|
|
||||||
return null
|
|
||||||
}
|
|
||||||
|
|
||||||
// Trova l'inizio e la fine dell'array di oggetti nell'input
|
|
||||||
const start = tagBlock.indexOf('[{');
|
|
||||||
const end = tagBlock.lastIndexOf('}]') + 2;
|
|
||||||
|
|
||||||
// Estrai la sottostringa contenente l'array di oggetti
|
|
||||||
const jsonString = tagBlock.substring(start, end);
|
|
||||||
|
|
||||||
// Parsea la stringa JSON in un array di oggetti
|
|
||||||
const videoRelatedArray = JSON.parse(jsonString);
|
|
||||||
|
|
||||||
// Mappa ogni oggetto nell'array per rinominare le chiavi
|
|
||||||
//@ts-ignore
|
|
||||||
const parsedArray = videoRelatedArray.map(obj => ({
|
|
||||||
//@ts-ignore
|
|
||||||
videoUrl: obj.u,
|
|
||||||
imgUrl: obj.i,
|
|
||||||
text: obj.tf
|
|
||||||
}));
|
|
||||||
|
|
||||||
return parsedArray;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const removeHttpS = (url: string): string => {
|
export const removeHttpS = (url: string): string => {
|
||||||
if (url.startsWith("http://")) {
|
if (url.startsWith("http://")) {
|
||||||
return url.slice(7);
|
return url.slice(7);
|
||||||
|
|
Loading…
Reference in New Issue