Release v0.2.0 #74

Merged
lamacchinadesiderante merged 24 commits from develop into main 2024-05-13 18:07:51 +00:00
11 changed files with 170 additions and 73 deletions
Showing only changes of commit 024b5eb206 - Show all commits

View File

@ -17,3 +17,8 @@ export interface VideoData {
hiResUrl?: string, hiResUrl?: string,
hlsUrl?: string hlsUrl?: string
} }
export interface VideoAgent {
getGallery(params?: FetchParams): Promise<GalleryData[]>
getVideo(id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]>
}

View File

@ -2,11 +2,13 @@ import { FetchParams, GalleryData, VideoData } from "@/meta/data";
import { Platforms } from "@/meta/settings"; import { Platforms } from "@/meta/settings";
import { fetchXVideosGalleryData } from "./scrape/xvideos/gallery"; import { XVideosAgent } from "./scrape/xvideos/agent";
import { fetchXvideosVideoData } from "./scrape/xvideos/video"; import { XNXXAgent } from "./scrape/xnxx/agent";
import { fetchXNXXGalleryData } from "./scrape/xnxx/gallery"; const AgentMapper = {
import { XNXX_BASE_URL } from "@/constants/urls"; [Platforms.xvideos]: XVideosAgent,
[Platforms.xnxx]: XNXXAgent
}
export class VideoAgent { export class VideoAgent {
platform: Platforms; platform: Platforms;
@ -16,25 +18,10 @@ export class VideoAgent {
} }
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => { public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
switch (this.platform) { return await new AgentMapper[this.platform]().getGallery(params)
case Platforms.xvideos:
return await fetchXVideosGalleryData(params)
case Platforms.xnxx:
return await fetchXNXXGalleryData(params)
default:
return []
}
} }
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => { public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
switch (this.platform) { return await new AgentMapper[this.platform]().getVideo(id, params)
case Platforms.xvideos:
return await fetchXvideosVideoData(id, params)
case Platforms.xnxx:
return await fetchXvideosVideoData(id, { ...params, baseUrl: XNXX_BASE_URL })
default:
return [{ lowResUrl: '' }, []]
} }
} }
}

View File

@ -1,5 +1,5 @@
import { XVIDEOS_BASE_URL } from "@/constants/urls"; import { XVIDEOS_BASE_URL } from "@/constants/urls";
import { removeHttpS } from "../string"; import { removeHttpS } from "@/utils/string";
const getRandomUserAgent = (): string => { const getRandomUserAgent = (): string => {

View File

@ -0,0 +1,44 @@
import { GalleryData } from "@/meta/data";
import { Platforms } from "@/meta/settings";
export const findVideoUrlInsideTagStringByFunctionNameAndExtension = (
tagBlock: string, functionName: string, extension: string): string | null => {
const start = tagBlock.indexOf(`html5player.${functionName}('`) + `html5player.${functionName}('`.length;
const end = tagBlock.toString().indexOf("'", start);
const substr = tagBlock.substring(start, end);
if (substr.includes(extension)) {
return substr
}
return null
}
export const findRelatedVideos = (tagBlock: string, platform: Platforms): GalleryData[] | null => {
if (!(tagBlock.includes('video_related=['))) {
return null
}
// Trova l'inizio e la fine dell'array di oggetti nell'input
const start = tagBlock.indexOf('[{');
const end = tagBlock.lastIndexOf('}]') + 2;
// Estrai la sottostringa contenente l'array di oggetti
const jsonString = tagBlock.substring(start, end);
// Parsea la stringa JSON in un array di oggetti
const videoRelatedArray = JSON.parse(jsonString);
// Mappa ogni oggetto nell'array per rinominare le chiavi
//@ts-ignore
const parsedArray = videoRelatedArray.map(obj => ({
//@ts-ignore
videoUrl: obj.u,
imgUrl: obj.i,
text: obj.tf,
platform
}));
return parsedArray;
}

View File

@ -0,0 +1,15 @@
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
import { fetchXNXXGalleryData, } from "./gallery";
import { fetchXNXXVideoData } from "./video";
export class XNXXAgent implements VideoAgent {
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
return await fetchXNXXGalleryData(params)
}
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
return await fetchXNXXVideoData(id, params)
}
}

View File

@ -3,7 +3,7 @@ import axios, { AxiosError } from 'axios';
import * as cheerio from "cheerio"; import * as cheerio from "cheerio";
import { getHeaders } from '../headers'; import { getHeaders } from '@/utils/scrape/common/headers';
import { getXNXXQueryUrl } from './url'; import { getXNXXQueryUrl } from './url';
import { Platforms } from '@/meta/settings'; import { Platforms } from '@/meta/settings';

View File

@ -0,0 +1,71 @@
import { XNXX_BASE_URL } from '@/constants/urls';
import { FetchParams, GalleryData, VideoData } from '@/meta/data';
import axios, { AxiosError } from 'axios';
import * as cheerio from "cheerio";
import { Platforms } from '@/meta/settings';
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
import { getHeaders } from '@/utils/scrape/common/headers';
export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
let data: VideoData = {
lowResUrl: ''
}
let related: GalleryData[] = [];
const host = XNXX_BASE_URL
const reqHeaders = getHeaders(host)
const queryUrl = `${host}${videoId}`
await axios.get(queryUrl, reqHeaders)
.then(response => {
const html = response.data;
const $ = cheerio.load(html);
const scriptTags = $("script");
// populate video data object
scriptTags.map((idx, elem) => {
const lowResUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoUrlLow', '.mp4')
const hiResUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoUrlHigh', '.mp4')
const hlsUrl = findVideoUrlInsideTagStringByFunctionNameAndExtension($(elem).toString(), 'setVideoHLS', '.m3u8')
if (lowResUrl) {
data.lowResUrl = lowResUrl;
}
if (hiResUrl) {
data.hiResUrl = hiResUrl
}
if (hlsUrl) {
data.hlsUrl = hlsUrl
}
})
// populate related gallery
scriptTags.map((idx, elem) => {
const relatedVideos = findRelatedVideos($(elem).toString(), Platforms.xnxx)
if (relatedVideos) {
related = relatedVideos
}
})
}).catch((error: AxiosError) => {
// handle errors
});
return [data, related];
}

View File

@ -0,0 +1,15 @@
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
import { fetchXVideosGalleryData } from "./gallery";
import { fetchXvideosVideoData } from "./video";
export class XVideosAgent implements VideoAgent {
public getGallery = async (params?: FetchParams): Promise<GalleryData[]> => {
return await fetchXVideosGalleryData(params)
}
public getVideo = async (id: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
return await fetchXvideosVideoData(id, params)
}
}

View File

@ -2,7 +2,7 @@ import { FetchParams, GalleryData } from '@/meta/data';
import axios, { AxiosError } from 'axios'; import axios, { AxiosError } from 'axios';
import * as cheerio from "cheerio"; import * as cheerio from "cheerio";
import { getHeaders } from '../headers'; import { getHeaders } from '@/utils/scrape/common/headers';
import { getXVideosQueryUrl } from './url'; import { getXVideosQueryUrl } from './url';
import { Platforms } from '@/meta/settings'; import { Platforms } from '@/meta/settings';

View File

@ -4,8 +4,10 @@ import { FetchParams, GalleryData, VideoData } from '@/meta/data';
import axios, { AxiosError } from 'axios'; import axios, { AxiosError } from 'axios';
import * as cheerio from "cheerio"; import * as cheerio from "cheerio";
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '../../string'; import { getHeaders } from '@/utils/scrape/common/headers';
import { getHeaders } from '../headers'; import { Platforms } from '@/meta/settings';
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => { export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
@ -15,9 +17,11 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
let related: GalleryData[] = []; let related: GalleryData[] = [];
const reqHeaders = getHeaders() const host = XVIDEOS_BASE_URL
const queryUrl = `${(params && params.baseUrl) ?? XVIDEOS_BASE_URL}${videoId}` const reqHeaders = getHeaders(host)
const queryUrl = `${host}${videoId}`
await axios.get(queryUrl, reqHeaders) await axios.get(queryUrl, reqHeaders)
@ -25,7 +29,6 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
const html = response.data; const html = response.data;
const $ = cheerio.load(html); const $ = cheerio.load(html);
const scriptTags = $("script"); const scriptTags = $("script");
@ -53,7 +56,7 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
// populate related gallery // populate related gallery
scriptTags.map((idx, elem) => { scriptTags.map((idx, elem) => {
const relatedVideos = findRelatedVideos($(elem).toString()) const relatedVideos = findRelatedVideos($(elem).toString(), Platforms.xvideos)
if (relatedVideos) { if (relatedVideos) {
related = relatedVideos related = relatedVideos

View File

@ -1,46 +1,3 @@
import { GalleryData } from "@/meta/data";
export const findVideoUrlInsideTagStringByFunctionNameAndExtension = (
tagBlock: string, functionName: string, extension: string): string | null => {
const start = tagBlock.indexOf(`html5player.${functionName}('`) + `html5player.${functionName}('`.length;
const end = tagBlock.toString().indexOf("'", start);
const substr = tagBlock.substring(start, end);
if (substr.includes(extension)) {
return substr
}
return null
}
export const findRelatedVideos = (tagBlock: string): GalleryData[] | null => {
if (!(tagBlock.includes('video_related=['))) {
return null
}
// Trova l'inizio e la fine dell'array di oggetti nell'input
const start = tagBlock.indexOf('[{');
const end = tagBlock.lastIndexOf('}]') + 2;
// Estrai la sottostringa contenente l'array di oggetti
const jsonString = tagBlock.substring(start, end);
// Parsea la stringa JSON in un array di oggetti
const videoRelatedArray = JSON.parse(jsonString);
// Mappa ogni oggetto nell'array per rinominare le chiavi
//@ts-ignore
const parsedArray = videoRelatedArray.map(obj => ({
//@ts-ignore
videoUrl: obj.u,
imgUrl: obj.i,
text: obj.tf
}));
return parsedArray;
}
export const removeHttpS = (url: string): string => { export const removeHttpS = (url: string): string => {
if (url.startsWith("http://")) { if (url.startsWith("http://")) {
return url.slice(7); return url.slice(7);