add redis integration (WIP)

docker-compose.yaml

@@ -19,3 +19,10 @@ services:
 
     ports:
       - "8069:3000"
+
+    env_file: .env
+
+  redis:
+      image: redis:alpine
+      container_name: redis
+      restart: unless-stopped

package-lock.json

@@ -22,6 +22,7 @@
         "react-icons": "^5.1.0",
         "react-image": "^4.1.0",
         "react-redux": "^9.1.1",
+        "redis": "^4.6.14",
         "redux-persist": "^6.0.0",
         "video.js": "^8.10.0",
         "videojs-hls-quality-selector": "^2.0.0"
@@ -30,6 +31,7 @@
         "@types/node": "^20",
         "@types/react": "^18",
         "@types/react-dom": "^18",
+        "@types/redis": "^4.0.11",
         "eslint": "^8",
         "eslint-config-next": "14.2.2",
         "sass": "^1.75.0",
@@ -477,6 +479,59 @@
         "node": ">=14"
       }
     },
+    "node_modules/@redis/bloom": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz",
+      "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/client": {
+      "version": "1.5.16",
+      "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.16.tgz",
+      "integrity": "sha512-X1a3xQ5kEMvTib5fBrHKh6Y+pXbeKXqziYuxOUo1ojQNECg4M5Etd1qqyhMap+lFUOAh8S7UYevgJHOm4A+NOg==",
+      "dependencies": {
+        "cluster-key-slot": "1.1.2",
+        "generic-pool": "3.9.0",
+        "yallist": "4.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
+    "node_modules/@redis/graph": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz",
+      "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/json": {
+      "version": "1.0.6",
+      "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz",
+      "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/search": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz",
+      "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
+    "node_modules/@redis/time-series": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz",
+      "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==",
+      "peerDependencies": {
+        "@redis/client": "^1.0.0"
+      }
+    },
     "node_modules/@reduxjs/toolkit": {
       "version": "2.2.3",
       "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.2.3.tgz",
@@ -572,6 +627,16 @@
         "@types/react": "*"
       }
     },
+    "node_modules/@types/redis": {
+      "version": "4.0.11",
+      "resolved": "https://registry.npmjs.org/@types/redis/-/redis-4.0.11.tgz",
+      "integrity": "sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg==",
+      "deprecated": "This is a stub types definition. redis provides its own type definitions, so you do not need this installed.",
+      "dev": true,
+      "dependencies": {
+        "redis": "*"
+      }
+    },
     "node_modules/@types/use-sync-external-store": {
       "version": "0.0.3",
       "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz",
@@ -1317,6 +1382,14 @@
       "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz",
       "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA=="
     },
+    "node_modules/cluster-key-slot": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz",
+      "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/color-convert": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -2462,6 +2535,14 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/generic-pool": {
+      "version": "3.9.0",
+      "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz",
+      "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==",
+      "engines": {
+        "node": ">= 4"
+      }
+    },
     "node_modules/get-intrinsic": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz",
@@ -4201,6 +4282,22 @@
         "node": ">=8.10.0"
       }
     },
+    "node_modules/redis": {
+      "version": "4.6.14",
+      "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.14.tgz",
+      "integrity": "sha512-GrNg/e33HtsQwNXL7kJT+iNFPSwE1IPmd7wzV3j4f2z0EYxZfZE7FVTmUysgAtqQQtg5NXF5SNLR9OdO/UHOfw==",
+      "workspaces": [
+        "./packages/*"
+      ],
+      "dependencies": {
+        "@redis/bloom": "1.2.0",
+        "@redis/client": "1.5.16",
+        "@redis/graph": "1.1.1",
+        "@redis/json": "1.0.6",
+        "@redis/search": "1.1.6",
+        "@redis/time-series": "1.0.5"
+      }
+    },
     "node_modules/redux": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz",
@@ -5306,8 +5403,7 @@
     "node_modules/yallist": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz",
-      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==",
+      "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A=="
-      "dev": true
     },
     "node_modules/yocto-queue": {
       "version": "0.1.0",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "proxyraye-next",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -23,6 +23,7 @@
     "react-icons": "^5.1.0",
     "react-image": "^4.1.0",
     "react-redux": "^9.1.1",
+    "redis": "^4.6.14",
     "redux-persist": "^6.0.0",
     "video.js": "^8.10.0",
     "videojs-hls-quality-selector": "^2.0.0"
@@ -31,6 +32,7 @@
     "@types/node": "^20",
     "@types/react": "^18",
     "@types/react-dom": "^18",
+    "@types/redis": "^4.0.11",
     "eslint": "^8",
     "eslint-config-next": "14.2.2",
     "sass": "^1.75.0",

src/constants/redis.ts

@@ -0,0 +1,4 @@
+export const DEFAULT_XVIDEOS_CONTENT_EXPIRY = { EX: 60 * 60 * 24 };
+export const DEFAULT_XNXX_CONTENT_EXPIRY = { EX: 60 * 60 * 24 };
+
+export const DEFAULT_RELATED_VIDEO_KEY_PATH = '/related/'

src/redis/client.ts

@@ -0,0 +1,38 @@
+import { createClient } from 'redis';
+
+export const getDataFromRedis = async (key: string): Promise<Object | null> => {
+
+    if (Boolean(process.env.ENABLE_REDIS)) {
+
+        const redis = await createClient({ url: process.env.REDIS_URL })
+            .on('error', err => console.log('Redis Client Error', err))
+            .connect()
+
+        const cached = await redis.get(key);
+
+        if (cached) {
+            await redis.disconnect();
+
+            return JSON.parse(cached)
+        } else {
+            return null
+        }
+
+
+    } else {
+        return null
+    }
+}
+
+export const storeDataIntoRedis = async (key: string, data: Object, expiry?: Object) => {
+    if (Boolean(process.env.ENABLE_REDIS)) {
+        const redis = await createClient({ url: process.env.REDIS_URL })
+            .on('error', err => console.log('Redis Client Error', err))
+            .connect()
+
+        await redis.set(key, JSON.stringify(data), expiry);
+
+        await redis.disconnect();
+    }
+
+} 

src/utils/scrape/xnxx/agent.ts

@@ -1,5 +1,5 @@
 import { FetchParams, GalleryData, VideoAgent, VideoData } from "@/meta/data";
-import { fetchXNXXGalleryData,  } from "./gallery";
+import { fetchXNXXGalleryData } from "./gallery";
 import { fetchXNXXVideoData } from "./video";
 
 export class XNXXAgent implements VideoAgent {

src/utils/scrape/xnxx/gallery.ts

@@ -8,6 +8,8 @@ import { getXNXXQueryUrl } from './url';
 
 import { Platforms } from '@/meta/settings';
 import { XNXX_BASE_URL } from '@/constants/urls';
+import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
+import { DEFAULT_XNXX_CONTENT_EXPIRY } from '@/constants/redis';
 
 export const fetchXNXXGalleryData = async (params?: FetchParams): Promise<GalleryData[]> => {
 
@@ -17,9 +19,15 @@ export const fetchXNXXGalleryData = async (params?: FetchParams): Promise<Galler
 
     const queryUrl = await getXNXXQueryUrl(params?.query)
 
+    const cachedData = await getDataFromRedis(queryUrl)
+
+    if (cachedData) {
+        return cachedData as GalleryData[]
+    }
+
     await axios.get(queryUrl, reqHeaders)
 
-        .then(response => {
+        .then(async response => {
 
             const html = response.data;
 
@@ -41,6 +49,8 @@ export const fetchXNXXGalleryData = async (params?: FetchParams): Promise<Galler
                 })
             })
 
+            await storeDataIntoRedis(queryUrl, data, DEFAULT_XNXX_CONTENT_EXPIRY);
+
         }).catch((error: AxiosError) => {
             // handle errors
         });

src/utils/scrape/xnxx/video.ts

@@ -8,6 +8,8 @@ import * as cheerio from "cheerio";
 import { Platforms } from '@/meta/settings';
 import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
 import { getHeaders } from '@/utils/scrape/common/headers';
+import { DEFAULT_RELATED_VIDEO_KEY_PATH, DEFAULT_XNXX_CONTENT_EXPIRY } from '@/constants/redis';
+import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
 
 export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
 
@@ -23,9 +25,16 @@ export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams):
 
     const queryUrl = `${host}${videoId}`
 
+    const cachedVideoData = await getDataFromRedis(queryUrl)
+    const cachedRelatedData = await getDataFromRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH)
+
+    if (cachedVideoData) {
+        return [cachedVideoData as VideoData, cachedRelatedData as GalleryData[] ?? []]
+    }
+
     await axios.get(queryUrl, reqHeaders)   
 
-        .then(response => {
+        .then(async response => {
 
             const html = response.data;
 
@@ -63,6 +72,9 @@ export const fetchXNXXVideoData = async (videoId: string, params?: FetchParams):
                 }
             })
 
+            await storeDataIntoRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH, related, DEFAULT_XNXX_CONTENT_EXPIRY);
+            await storeDataIntoRedis(queryUrl, data, DEFAULT_XNXX_CONTENT_EXPIRY);
+
         }).catch((error: AxiosError) => {
             // handle errors
         });

src/utils/scrape/xvideos/gallery.ts

@@ -6,6 +6,9 @@ import { getHeaders } from '@/utils/scrape/common/headers';
 import { getXVideosQueryUrl } from './url';
 import { Platforms } from '@/meta/settings';
 
+import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
+import { DEFAULT_XVIDEOS_CONTENT_EXPIRY } from '@/constants/redis';
+
 export const fetchXVideosGalleryData = async (params?: FetchParams): Promise<GalleryData[]> => {
 
     let data: GalleryData[] = [];
@@ -14,22 +17,28 @@ export const fetchXVideosGalleryData = async (params?: FetchParams): Promise<Gal
 
     const queryUrl = await getXVideosQueryUrl(params?.query)
 
+    const cachedData = await getDataFromRedis(queryUrl)
+
+    if (cachedData) {
+        return cachedData as GalleryData[]
+    }
+
     await axios.get(queryUrl, reqHeaders)
 
-        .then(response => {
+        .then(async response => {
 
             const html = response.data;
 
             const $ = cheerio.load(html);
-        
+
             const thumbs = $(".thumb-block");
-        
+
             thumbs.map((key, thumb) => {
-        
+
                 const videoUrl = $(thumb).find(".thumb a").attr("href")
                 const imgUrl = $(thumb).find(".thumb img").attr("data-src")
                 const text = $(thumb).find(".thumb-under a").attr("title")
-        
+
                 videoUrl && imgUrl && text && data.push({
                     videoUrl,
                     imgUrl,
@@ -38,7 +47,10 @@ export const fetchXVideosGalleryData = async (params?: FetchParams): Promise<Gal
                 })
             })
 
-        }).catch((error: AxiosError) => {
+            await storeDataIntoRedis(queryUrl, data, DEFAULT_XVIDEOS_CONTENT_EXPIRY);
+
+        })
+        .catch((error: AxiosError) => {
             // handle errors
         });
 

src/utils/scrape/xvideos/video.ts

@@ -8,6 +8,8 @@ import { getHeaders } from '@/utils/scrape/common/headers';
 import { Platforms } from '@/meta/settings';
 
 import { findRelatedVideos, findVideoUrlInsideTagStringByFunctionNameAndExtension } from '@/utils/scrape/common/wgcz';
+import { getDataFromRedis, storeDataIntoRedis } from '@/redis/client';
+import { DEFAULT_RELATED_VIDEO_KEY_PATH, DEFAULT_XVIDEOS_CONTENT_EXPIRY } from '@/constants/redis';
 
 export const fetchXvideosVideoData = async (videoId: string, params?: FetchParams): Promise<[VideoData, GalleryData[]]> => {
 
@@ -23,9 +25,16 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
 
     const queryUrl = `${host}${videoId}`
 
+    const cachedVideoData = await getDataFromRedis(queryUrl)
+    const cachedRelatedData = await getDataFromRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH)
+
+    if (cachedVideoData) {
+        return [cachedVideoData as VideoData, cachedRelatedData as GalleryData[] ?? []]
+    }
+
     await axios.get(queryUrl, reqHeaders)   
 
-        .then(response => {
+        .then(async response => {
 
             const html = response.data;
 
@@ -63,6 +72,9 @@ export const fetchXvideosVideoData = async (videoId: string, params?: FetchParam
                 }
             })
 
+            await storeDataIntoRedis(queryUrl + DEFAULT_RELATED_VIDEO_KEY_PATH, related, DEFAULT_XVIDEOS_CONTENT_EXPIRY);
+            await storeDataIntoRedis(queryUrl, data, DEFAULT_XVIDEOS_CONTENT_EXPIRY);
+
         }).catch((error: AxiosError) => {
             // handle errors
         });