lamacchinadesiderante/proxyraye-nextjs
lamacchinadesiderante
/
proxyraye-nextjs
0
0
Fork 0
Code Issues 20 Pull Requests Packages Projects 6 Releases 7 Wiki Activity

v0.3/redis-setup #80

Merged
lamacchinadesiderante merged 7 commits from v0.3/redis-setup into develop 2024-05-18 11:32:14 +00:00
Conversation 0 Commits 7 Files Changed 15 +206 -13
10 changed files with 206 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a1d751e0bb - Show all commits

7
docker-compose.yaml
View File

@ -19,3 +19,10 @@ services:
ports:
- "8069:3000"
env_file: .env
redis:
image: redis:alpine
container_name: redis
restart: unless-stopped

100
package-lock.json generated
View File

@ -22,6 +22,7 @@
"react-icons": "^5.1.0",
"react-image": "^4.1.0",
"react-redux": "^9.1.1",
"redis": "^4.6.14",
"redux-persist": "^6.0.0",
"video.js": "^8.10.0",
"videojs-hls-quality-selector": "^2.0.0"
@ -30,6 +31,7 @@
"@types/node": "^20",
"@types/react": "^18",
"@types/react-dom": "^18",
"@types/redis": "^4.0.11",
"eslint": "^8",
"eslint-config-next": "14.2.2",
"sass": "^1.75.0",
@ -477,6 +479,59 @@
"node": ">=14"
}
},
"node_modules/@redis/bloom": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
"integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/client": {
"version": "1.5.16",
"resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.16.tgz",
"integrity": "sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==",
"dependencies": {
"cluster-key-slot": "1.1.2",
"generic-pool": "3.9.0",
"yallist": "4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/@redis/graph": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
"integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/json": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
"integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/search": {
"version": "1.1.6",
"resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz",
"integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@redis/time-series": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
"integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
"peerDependencies": {
"@redis/client": "^1.0.0"
}
},
"node_modules/@reduxjs/toolkit": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.3.tgz",
@ -572,6 +627,16 @@
"@types/react": "*"
}
},
"node_modules/@types/redis": {
"version": "4.0.11",
"resolved": "https://registry.npmjs.org/@types/redis/-/redis-4.0.11.tgz",
"integrity": "sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg==",
"deprecated": "This is a stub types definition. redis provides its own type definitions, so you do not need this installed.",
"dev": true,
"dependencies": {
"redis": "*"
}
},
"node_modules/@types/use-sync-external-store": {
"version": "0.0.3",
"resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz",
@ -1317,6 +1382,14 @@
"resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
"integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
},
"node_modules/cluster-key-slot": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
"integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@ -2462,6 +2535,14 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/generic-pool": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
"integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
"engines": {
"node": ">= 4"
}
},
"node_modules/get-intrinsic": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
@ -4201,6 +4282,22 @@
"node": ">=8.10.0"
}
},
"node_modules/redis": {
"version": "4.6.14",
"resolved": "https://registry.npmjs.org/redis/-/redis-4.6.14.tgz",
"integrity": "sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==",
"workspaces": [
"./packages/*"
],
"dependencies": {
"@redis/bloom": "1.2.0",
"@redis/client": "1.5.16",
"@redis/graph": "1.1.1",
"@redis/json": "1.0.6",
"@redis/search": "1.1.6",
"@redis/time-series": "1.0.5"
}
},
"node_modules/redux": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz",
@ -5306,8 +5403,7 @@
"node_modules/yallist": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
"dev": true
"integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
},
"node_modules/yocto-queue": {
"version": "0.1.0",

4
package.json
View File

@ -1,6 +1,6 @@
{
"name": "proxyraye-next",
"version": "0.2.0",
"version": "0.3.0",
"private": true,
"scripts": {
"dev": "next dev",
@ -23,6 +23,7 @@
"react-icons": "^5.1.0",
"react-image": "^4.1.0",
"react-redux": "^9.1.1",
"redis": "^4.6.14",
"redux-persist": "^6.0.0",
"video.js": "^8.10.0",
"videojs-hls-quality-selector": "^2.0.0"
@ -31,6 +32,7 @@
"@types/node": "^20",
"@types/react": "^18",
"@types/react-dom": "^18",
"@types/redis": "^4.0.11",
"eslint": "^8",
"eslint-config-next": "14.2.2",
"sass": "^1.75.0",

4
src/constants/redis.ts Normal file
View File

@ -0,0 +1,4 @@
export const DEFAULT_XVIDEOS_CONTENT_EXPIRY = { EX: 60 * 60 * 24 };
export const DEFAULT_XNXX_CONTENT_EXPIRY = { EX: 60 * 60 * 24 };
export const DEFAULT_RELATED_VIDEO_KEY_PATH = '/related/'

38
src/redis/client.ts Normal file
View File

@ -0,0 +1,38 @@
import { createClient } from 'redis';
export const getDataFromRedis = async (key: string): Promise<Object | null> => {
if (Boolean(process.env.ENABLE_REDIS)) {
const redis = await createClient({ url: process.env.REDIS_URL })
.on('error', err => console.log('Redis Client Error', err))
.connect()
const cached = await redis.get(key);
if (cached) {
await redis.disconnect();
return JSON.parse(cached)
} else {
return null
}
} else {
return null
}
}
export const storeDataIntoRedis = async (key: string, data: Object, expiry?: Object) => {
if (Boolean(process.env.ENABLE_REDIS)) {
const redis = await createClient({ url: process.env.REDIS_URL })
.on('error', err => console.log('Redis Client Error', err))
.connect()
await redis.set(key, JSON.stringify(data), expiry);
await redis.disconnect();
}
}

2
src/utils/scrape/xnxx/agent.ts
View File

@ -1,5 +1,5 @@
import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
import { fetchXNXXGalleryData, } from "./gallery";
import { fetchXNXXGalleryData } from "./gallery";
import { fetchXNXXVideoData } from "./video";
export class XNXXAgent implements VideoAgent {

12
src/utils/scrape/xnxx/gallery.ts
View File

@ -8,6 +8,8 @@ import { getXNXXQueryUrl } from './url';
import { Platforms } from '@/meta/settings';
import { XNXX_BASE_URL } from '@/constants/urls';
import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
import { DEFAULT_XNXX_CONTENT_EXPIRY } from '@/constants/redis';
export const fetchXNXXGalleryData = async (params?: FetchParams): Promise<GalleryData[]> => {
@ -17,9 +19,15 @@ export const fetchXNXXGalleryData = async (params?: FetchParams): Promise<Galler
const queryUrl = await getXNXXQueryUrl(params?.query)
const cachedData = await getDataFromRedis(queryUrl)
if (cachedData) {
return cachedData as GalleryData[]
}
await axios.get(queryUrl, reqHeaders)
.then(response => {
.then(async response => {
const html = response.data;
@ -41,6 +49,8 @@ export const fetchXNXXGalleryData = async (params?: FetchParams): Promise<Galler
})
})
await storeDataIntoRedis(queryUrl, data, DEFAULT_XNXX_CONTENT_EXPIRY);
}).catch((error: AxiosError) => {
// handle errors
});

14
src/utils/scrape/xnxx/video.ts
View File

@ -8,6 +8,8 @@ import * as cheerio from "cheerio";
import { Platforms } from '@/meta/settings';
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
import { getHeaders } from '@/utils/scrape/common/headers';
import { DEFAULT_RELATED_VIDEO_KEY_PATH, DEFAULT_XNXX_CONTENT_EXPIRY } from '@/constants/redis';
import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
@ -23,9 +25,16 @@ export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams):
const queryUrl = `${host}${videoId}`
const cachedVideoData = await getDataFromRedis(queryUrl)
const cachedRelatedData = await getDataFromRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH)
if (cachedVideoData) {
return [cachedVideoData as VideoData, cachedRelatedData as GalleryData[] ?? []]
}
await axios.get(queryUrl, reqHeaders)
.then(response => {
.then(async response => {
const html = response.data;
@ -63,6 +72,9 @@ export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams):
}
})
await storeDataIntoRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH, related, DEFAULT_XNXX_CONTENT_EXPIRY);
await storeDataIntoRedis(queryUrl, data, DEFAULT_XNXX_CONTENT_EXPIRY);
}).catch((error: AxiosError) => {
// handle errors
});

24
src/utils/scrape/xvideos/gallery.ts
View File

@ -6,6 +6,9 @@ import { getHeaders } from '@/utils/scrape/common/headers';
import { getXVideosQueryUrl } from './url';
import { Platforms } from '@/meta/settings';
import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
import { DEFAULT_XVIDEOS_CONTENT_EXPIRY } from '@/constants/redis';
export const fetchXVideosGalleryData = async (params?: FetchParams): Promise<GalleryData[]> => {
let data: GalleryData[] = [];
@ -14,22 +17,28 @@ export const fetchXVideosGalleryData = async (params?: FetchParams): Promise<Gal
const queryUrl = await getXVideosQueryUrl(params?.query)
const cachedData = await getDataFromRedis(queryUrl)
if (cachedData) {
return cachedData as GalleryData[]
}
await axios.get(queryUrl, reqHeaders)
.then(response => {
.then(async response => {
const html = response.data;
const $ = cheerio.load(html);
const thumbs = $(".thumb-block");
thumbs.map((key, thumb) => {
const videoUrl = $(thumb).find(".thumb a").attr("href")
const imgUrl = $(thumb).find(".thumb img").attr("data-src")
const text = $(thumb).find(".thumb-under a").attr("title")
videoUrl && imgUrl && text && data.push({
videoUrl,
imgUrl,
@ -38,7 +47,10 @@ export const fetchXVideosGalleryData = async (params?: FetchParams): Promise<Gal
})
})
}).catch((error: AxiosError) => {
await storeDataIntoRedis(queryUrl, data, DEFAULT_XVIDEOS_CONTENT_EXPIRY);
})
.catch((error: AxiosError) => {
// handle errors
});

14
src/utils/scrape/xvideos/video.ts
View File

@ -8,6 +8,8 @@ import { getHeaders } from '@/utils/scrape/common/headers';
import { Platforms } from '@/meta/settings';
import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
import { DEFAULT_RELATED_VIDEO_KEY_PATH, DEFAULT_XVIDEOS_CONTENT_EXPIRY } from '@/constants/redis';
export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
@ -23,9 +25,16 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
const queryUrl = `${host}${videoId}`
const cachedVideoData = await getDataFromRedis(queryUrl)
const cachedRelatedData = await getDataFromRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH)
if (cachedVideoData) {
return [cachedVideoData as VideoData, cachedRelatedData as GalleryData[] ?? []]
}
await axios.get(queryUrl, reqHeaders)
.then(response => {
.then(async response => {
const html = response.data;
@ -63,6 +72,9 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
}
})
await storeDataIntoRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH, related, DEFAULT_XVIDEOS_CONTENT_EXPIRY);
await storeDataIntoRedis(queryUrl, data, DEFAULT_XVIDEOS_CONTENT_EXPIRY);
}).catch((error: AxiosError) => {
// handle errors
});