From 5fd0d9218707797999fe49e256244bb5cb8d2f66 Mon Sep 17 00:00:00 2001 From: zyachel Date: Sun, 13 Nov 2022 17:29:47 +0530 Subject: [PATCH] fix(redis): fix logs being polluted when redis is disabled also used streaming when redis is disabled for faster response. --- .env.local.example | 22 +++++---- .gitignore | 8 +++- package.json | 1 - pnpm-lock.yaml | 59 +++++++++++++++++++++++- src/components/title/Media.tsx | 33 ++++++------- src/pages/api/media_proxy.ts | 84 +++++++++++++++++++--------------- src/utils/redis.ts | 14 +++--- 7 files changed, 148 insertions(+), 73 deletions(-) diff --git a/.env.local.example b/.env.local.example index 261f5a3..2429dc8 100644 --- a/.env.local.example +++ b/.env.local.example @@ -1,15 +1,19 @@ -# required fields -# used for meta tags. e.g: 'https://libremdb.iket.me' don't add end slash. +### required fields +## used for meta tags. e.g: 'https://libremdb.iket.me'. don't add end slash. NEXT_PUBLIC_URL= -# optional fields. uncomment them and add the values if you wish so. -# default useragent for requesting data from imdb is 'axios/0.27.2' +### optional fields. modify according to your needs. +## comment it out if you wish to enable nextjs stats collection. more at https://nextjs.org/telemetry +NEXT_TELEMETRY_DISABLED=1 +## default useragent for requesting data from imdb is 'axios/0.27.2' # AXIOS_USERAGENT= -# default accept header is 'application/json, text/plain, */*' +## default accept header is 'application/json, text/plain, */*' # AXIOS_ACCEPT= +## for forcing a certain language for data we get from imdb +# AXIOS_LANGUAGE=en-US -# for docker, just set the domain to the container name, default is 'libremdb_redis' +### REDIS +## if you want to use redis to speed up the media proxy, set this to true +# USE_REDIS=true +## for docker, just set the domain to the container name, default is 'libremdb_redis' REDIS_URL=localhost:6379 - -# if you want to use redis to speed up the media proxy, set this to true -USE_REDIS = true \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7888140..fdab8f5 100644 --- a/.gitignore +++ b/.gitignore @@ -28,11 +28,15 @@ yarn-error.log* # typescript *.tsbuildinfo - next-env.d.ts +next-env.d.ts #just dev stuff dev/* + +# other lockfiles yarn.lock +package-lock.json # docker -docker-compose.yml \ No newline at end of file +docker-compose.yml +dump.rdb \ No newline at end of file diff --git a/package.json b/package.json index 917768b..a219a60 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,6 @@ "cheerio": "1.0.0-rc.12", "ioredis": "^5.2.3", "next": "12.2.5", - "node-fetch": "^3.2.10", "react": "18.2.0", "react-dom": "18.2.0", "sharp": "^0.31.0" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0532c6c..b4bad5c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -8,6 +8,7 @@ specifiers: cheerio: 1.0.0-rc.12 eslint: 8.22.0 eslint-config-next: 12.2.5 + ioredis: ^5.2.3 next: 12.2.5 react: 18.2.0 react-dom: 18.2.0 @@ -18,6 +19,7 @@ specifiers: dependencies: axios: 0.27.2 cheerio: 1.0.0-rc.12 + ioredis: 5.2.4 next: 12.2.5_ivfob5dyaiglqb5g2zdrumbbbm react: 18.2.0 react-dom: 18.2.0_react@18.2.0 @@ -85,6 +87,10 @@ packages: resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==} dev: true + /@ioredis/commands/1.2.0: + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + dev: false + /@next/env/12.2.5: resolution: {integrity: sha512-vLPLV3cpPGjUPT3PjgRj7e3nio9t6USkuew3JE/jMeon/9Mvp1WyR18v3iwnCuX7eUAm1HmAbJHHLAbcu/EJcw==} dev: false @@ -561,6 +567,11 @@ packages: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} dev: false + /cluster-key-slot/1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + dev: false + /color-convert/2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -665,7 +676,6 @@ packages: optional: true dependencies: ms: 2.1.2 - dev: true /decompress-response/6.0.0: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} @@ -696,6 +706,11 @@ packages: engines: {node: '>=0.4.0'} dev: false + /denque/2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dev: false + /detect-libc/2.0.1: resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==} engines: {node: '>=8'} @@ -1381,6 +1396,23 @@ packages: side-channel: 1.0.4 dev: true + /ioredis/5.2.4: + resolution: {integrity: sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==} + engines: {node: '>=12.22.0'} + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.4 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + dev: false + /is-arrayish/0.3.2: resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==} dev: false @@ -1545,6 +1577,14 @@ packages: p-locate: 5.0.0 dev: true + /lodash.defaults/4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + dev: false + + /lodash.isarguments/3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + dev: false + /lodash.merge/4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} dev: true @@ -1610,7 +1650,6 @@ packages: /ms/2.1.2: resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==} - dev: true /ms/2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1941,6 +1980,18 @@ packages: dependencies: picomatch: 2.3.1 + /redis-errors/1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + dev: false + + /redis-parser/3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + dependencies: + redis-errors: 1.2.0 + dev: false + /regenerator-runtime/0.13.9: resolution: {integrity: sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==} dev: true @@ -2093,6 +2144,10 @@ packages: resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==} engines: {node: '>=0.10.0'} + /standard-as-callback/2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + dev: false + /string.prototype.matchall/4.0.7: resolution: {integrity: sha512-f48okCX7JiwVi1NXCVWcFnZgADDC/n2vePlQ/KUCNqCikLLilQvwjMO8+BHVKvgzH0JB0J9LEPgxOGT02RoETg==} dependencies: diff --git a/src/components/title/Media.tsx b/src/components/title/Media.tsx index 117ed39..e2c63ed 100644 --- a/src/components/title/Media.tsx +++ b/src/components/title/Media.tsx @@ -1,16 +1,16 @@ -import Image from 'next/future/image' -import Link from 'next/link' -import { NextRouter } from 'next/router' -import { Media } from '../../interfaces/shared/title' -import { getProxiedIMDbImgUrl, modifyIMDbImg } from '../../utils/helpers' +import Image from 'next/future/image'; +import Link from 'next/link'; +import { NextRouter } from 'next/router'; +import { Media } from '../../interfaces/shared/title'; +import { getProxiedIMDbImgUrl, modifyIMDbImg } from '../../utils/helpers'; -import styles from '../../styles/modules/components/title/media.module.scss' +import styles from '../../styles/modules/components/title/media.module.scss'; type Props = { - className: string - media: Media - router: NextRouter -} + className: string; + media: Media; + router: NextRouter; +}; const Media = ({ className, media, router }: Props) => { return ( @@ -32,8 +32,9 @@ const Media = ({ className, media, router }: Props) => { modifyIMDbImg(media.trailer.thumbnail) )} className={styles.trailer__video} + preload="none" > - {media.trailer.urls.map((source) => ( + {media.trailer.urls.map(source => ( { )} {!!media.videos.total && - media.videos.videos.map((video) => ( + media.videos.videos.map(video => ( {

Images

- {media.images.images.map((image) => ( + {media.images.images.map(image => (
{
)} - ) -} -export default Media + ); +}; +export default Media; diff --git a/src/pages/api/media_proxy.ts b/src/pages/api/media_proxy.ts index f35b3be..5d8e6e2 100644 --- a/src/pages/api/media_proxy.ts +++ b/src/pages/api/media_proxy.ts @@ -1,59 +1,71 @@ -import { NextApiRequest, NextApiResponse } from 'next' -import redis from '../../utils/redis' -import axiosInstance from '../../utils/axiosInstance' -import { AxiosResponse } from 'axios' +import { NextApiRequest, NextApiResponse } from 'next'; +import redis from '../../utils/redis'; +import axiosInstance from '../../utils/axiosInstance'; const regex = - /^https:\/\/((m\.)?media-amazon\.com|imdb-video\.media-imdb\.com).*\.(jpg|jpeg|png|mp4|gif|webp).*$/ + /^https:\/\/((m\.)?media-amazon\.com|imdb-video\.media-imdb\.com).*\.(jpg|jpeg|png|mp4|gif|webp).*$/; export default async function handler( req: NextApiRequest, res: NextApiResponse ) { - const mediaUrl = req.query.url as string | undefined + try { + const mediaUrl = req.query.url as string | undefined; - if (!mediaUrl || !regex.test(mediaUrl)) - return res.status(400).json({ - success: false, - message: 'Invalid query', - }) + // 1. returning if query is illegal + if (!mediaUrl || !regex.test(mediaUrl)) + return res.status(400).json({ + success: false, + message: 'Invalid query', + }); - if (process.env.USE_REDIS === 'true') { - const cachedMedia = await redis.getBuffer(mediaUrl) + // 2. sending streamed response if redis isn't enabled + if (redis === null) { + const mediaRes = await axiosInstance.get(mediaUrl, { + responseType: 'stream', + }); + + res.setHeader('Content-Type', mediaRes.headers['content-type']); + mediaRes.data.pipe(res); + return; + } + + // 3. else if resourced is cached, sending it + const cachedMedia = await redis!.getBuffer(mediaUrl); if (cachedMedia) { - res.setHeader('x-cached', 'true') - res.status(302).send(cachedMedia) - return + res.setHeader('x-cached', 'true'); + res.status(302).send(cachedMedia); + return; } - } - let mediaRes: AxiosResponse - try { - mediaRes = await axiosInstance(mediaUrl, { responseType: 'arraybuffer' }) + // 4. else getting, caching and sending response + const mediaRes = await axiosInstance(mediaUrl, { + responseType: 'arraybuffer', + }); + + const data = mediaRes.data; + + // saving in redis for 30 minutes + await redis!.setex(mediaUrl, 30 * 60, Buffer.from(data)); + + // sending media + res.setHeader('x-cached', 'false'); + res.send(data); + + // sending token response on any error } catch { - res.status(404) + res.status(404); res.json({ success: false, - message: 'Error from IMDb', - }) - return + message: 'something went wrong', + }); + return; } - - const data = mediaRes.data - - if (process.env.USE_REDIS === 'true') { - // save in redis for 30 minutes - await redis.setex(mediaUrl, 30 * 60, Buffer.from(data)) - } - - // send media - res.setHeader('x-cached', 'false') - res.send(data) } export const config = { api: { responseLimit: false, }, -} +}; diff --git a/src/utils/redis.ts b/src/utils/redis.ts index eb87c77..1a1e17b 100644 --- a/src/utils/redis.ts +++ b/src/utils/redis.ts @@ -1,11 +1,11 @@ -import Redis from 'ioredis' +import Redis from 'ioredis'; -const redisUrl = process.env.REDIS_URL +const redisUrl = process.env.REDIS_URL; +const toUseRedis = process.env.USE_REDIS === 'true'; -if (!redisUrl) { - throw 'Please set the REDIS_URL environment variable.' -} +let redis: Redis | null; -const redis = new Redis(redisUrl) +if (toUseRedis && redisUrl) redis = new Redis(redisUrl); +else redis = null; -export default redis +export default redis;