fix(redis): fix logs being polluted when redis is disabled

also used streaming when redis is disabled for faster response.
2022-11-13 17:29:47 +05:30 · 2022-11-13 17:29:47 +05:30 · 5fd0d92187
commit 5fd0d92187
parent 6f664d2164
7 changed files with 148 additions and 73 deletions
--- a/.env.local.example
+++ b/.env.local.example
@ -1,15 +1,19 @@
-# required fields
-# used for meta tags. e.g: 'https://libremdb.iket.me' don't add end slash.
+### required fields
+## used for meta tags. e.g: 'https://libremdb.iket.me'. don't add end slash.
 NEXT_PUBLIC_URL=

-# optional fields. uncomment them and add the values if you wish so.
-# default useragent for requesting data from imdb is 'axios/0.27.2'
+### optional fields. modify according to your needs.
+## comment it out if you wish to enable nextjs stats collection. more at https://nextjs.org/telemetry
+NEXT_TELEMETRY_DISABLED=1
+## default useragent for requesting data from imdb is 'axios/0.27.2'
 # AXIOS_USERAGENT=
-# default accept header is 'application/json, text/plain, */*'
+## default accept header is 'application/json, text/plain, */*'
 # AXIOS_ACCEPT=
+## for forcing a certain language for data we get from imdb
+# AXIOS_LANGUAGE=en-US

-# for docker, just set the domain to the container name, default is 'libremdb_redis'
+### REDIS
+## if you want to use redis to speed up the media proxy, set this to true
+# USE_REDIS=true
+## for docker, just set the domain to the container name, default is 'libremdb_redis'
 REDIS_URL=localhost:6379
-
-# if you want to use redis to speed up the media proxy, set this to true
-USE_REDIS = true
--- a/.gitignore
+++ b/.gitignore
@ -28,11 +28,15 @@ yarn-error.log*

 # typescript
 *.tsbuildinfo
- next-env.d.ts
+next-env.d.ts

 #just dev stuff
 dev/*
+
+# other lockfiles 
 yarn.lock
+package-lock.json

 # docker
 docker-compose.yml
+dump.rdb
--- a/package.json
+++ b/package.json
@ -17,7 +17,6 @@
    "cheerio": "1.0.0-rc.12",
    "ioredis": "^5.2.3",
    "next": "12.2.5",
-    "node-fetch": "^3.2.10",
    "react": "18.2.0",
    "react-dom": "18.2.0",
    "sharp": "^0.31.0"
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -8,6 +8,7 @@ specifiers:
  cheerio: 1.0.0-rc.12
  eslint: 8.22.0
  eslint-config-next: 12.2.5
+  ioredis: ^5.2.3
  next: 12.2.5
  react: 18.2.0
  react-dom: 18.2.0
@ -18,6 +19,7 @@ specifiers:
 dependencies:
  axios: 0.27.2
  cheerio: 1.0.0-rc.12
+  ioredis: 5.2.4
  next: 12.2.5_ivfob5dyaiglqb5g2zdrumbbbm
  react: 18.2.0
  react-dom: 18.2.0_react@18.2.0
@ -85,6 +87,10 @@ packages:
    resolution: {integrity: sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==}
    dev: true

+  /@ioredis/commands/1.2.0:
+    resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
+    dev: false
+
  /@next/env/12.2.5:
    resolution: {integrity: sha512-vLPLV3cpPGjUPT3PjgRj7e3nio9t6USkuew3JE/jMeon/9Mvp1WyR18v3iwnCuX7eUAm1HmAbJHHLAbcu/EJcw==}
    dev: false
@ -561,6 +567,11 @@ packages:
    resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
    dev: false

+  /cluster-key-slot/1.1.2:
+    resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
+    engines: {node: '>=0.10.0'}
+    dev: false
+
  /color-convert/2.0.1:
    resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==}
    engines: {node: '>=7.0.0'}
@ -665,7 +676,6 @@ packages:
        optional: true
    dependencies:
      ms: 2.1.2
-    dev: true

  /decompress-response/6.0.0:
    resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==}
@ -696,6 +706,11 @@ packages:
    engines: {node: '>=0.4.0'}
    dev: false

+  /denque/2.1.0:
+    resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
+    engines: {node: '>=0.10'}
+    dev: false
+
  /detect-libc/2.0.1:
    resolution: {integrity: sha512-463v3ZeIrcWtdgIg6vI6XUncguvr2TnGl4SzDXinkt9mSLpBJKXT3mW6xT3VQdDN11+WVs29pgvivTc4Lp8v+w==}
    engines: {node: '>=8'}
@ -1381,6 +1396,23 @@ packages:
      side-channel: 1.0.4
    dev: true

+  /ioredis/5.2.4:
+    resolution: {integrity: sha512-qIpuAEt32lZJQ0XyrloCRdlEdUUNGG9i0UOk6zgzK6igyudNWqEBxfH6OlbnOOoBBvr1WB02mm8fR55CnikRng==}
+    engines: {node: '>=12.22.0'}
+    dependencies:
+      '@ioredis/commands': 1.2.0
+      cluster-key-slot: 1.1.2
+      debug: 4.3.4
+      denque: 2.1.0
+      lodash.defaults: 4.2.0
+      lodash.isarguments: 3.1.0
+      redis-errors: 1.2.0
+      redis-parser: 3.0.0
+      standard-as-callback: 2.1.0
+    transitivePeerDependencies:
+      - supports-color
+    dev: false
+
  /is-arrayish/0.3.2:
    resolution: {integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==}
    dev: false
@ -1545,6 +1577,14 @@ packages:
      p-locate: 5.0.0
    dev: true

+  /lodash.defaults/4.2.0:
+    resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
+    dev: false
+
+  /lodash.isarguments/3.1.0:
+    resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
+    dev: false
+
  /lodash.merge/4.6.2:
    resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
    dev: true
@ -1610,7 +1650,6 @@ packages:

  /ms/2.1.2:
    resolution: {integrity: sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==}
-    dev: true

  /ms/2.1.3:
    resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@ -1941,6 +1980,18 @@ packages:
    dependencies:
      picomatch: 2.3.1

+  /redis-errors/1.2.0:
+    resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
+    engines: {node: '>=4'}
+    dev: false
+
+  /redis-parser/3.0.0:
+    resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
+    engines: {node: '>=4'}
+    dependencies:
+      redis-errors: 1.2.0
+    dev: false
+
  /regenerator-runtime/0.13.9:
    resolution: {integrity: sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==}
    dev: true
@ -2093,6 +2144,10 @@ packages:
    resolution: {integrity: sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==}
    engines: {node: '>=0.10.0'}

+  /standard-as-callback/2.1.0:
+    resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
+    dev: false
+
  /string.prototype.matchall/4.0.7:
    resolution: {integrity: sha512-f48okCX7JiwVi1NXCVWcFnZgADDC/n2vePlQ/KUCNqCikLLilQvwjMO8+BHVKvgzH0JB0J9LEPgxOGT02RoETg==}
    dependencies:
--- a/src/components/title/Media.tsx
+++ b/src/components/title/Media.tsx
@ -1,16 +1,16 @@
-import Image from 'next/future/image'
-import Link from 'next/link'
-import { NextRouter } from 'next/router'
-import { Media } from '../../interfaces/shared/title'
-import { getProxiedIMDbImgUrl, modifyIMDbImg } from '../../utils/helpers'
+import Image from 'next/future/image';
+import Link from 'next/link';
+import { NextRouter } from 'next/router';
+import { Media } from '../../interfaces/shared/title';
+import { getProxiedIMDbImgUrl, modifyIMDbImg } from '../../utils/helpers';

-import styles from '../../styles/modules/components/title/media.module.scss'
+import styles from '../../styles/modules/components/title/media.module.scss';

 type Props = {
-  className: string
-  media: Media
-  router: NextRouter
-}
+  className: string;
+  media: Media;
+  router: NextRouter;
+};

 const Media = ({ className, media, router }: Props) => {
  return (
@ -32,8 +32,9 @@ const Media = ({ className, media, router }: Props) => {
                    modifyIMDbImg(media.trailer.thumbnail)
                  )}
                  className={styles.trailer__video}
+                  preload="none"
                >
-                  {media.trailer.urls.map((source) => (
+                  {media.trailer.urls.map(source => (
                    <source
                      key={source.url}
                      type={source.mimeType}
@ -46,7 +47,7 @@ const Media = ({ className, media, router }: Props) => {
            )}

            {!!media.videos.total &&
-              media.videos.videos.map((video) => (
+              media.videos.videos.map(video => (
                <Link href={`/video/${video.id}`} key={video.id}>
                  <a className={styles.video}>
                    <Image
@ -69,7 +70,7 @@ const Media = ({ className, media, router }: Props) => {
        <section className={styles.images}>
          <h2 className="heading heading__secondary">Images</h2>
          <div className={styles.images__container}>
-            {media.images.images.map((image) => (
+            {media.images.images.map(image => (
              <figure key={image.id} className={styles.image}>
                <Image
                  className={styles.image__img}
@ -87,6 +88,6 @@ const Media = ({ className, media, router }: Props) => {
        </section>
      )}
    </div>
-  )
-}
-export default Media
+  );
+};
+export default Media;
--- a/src/pages/api/media_proxy.ts
+++ b/src/pages/api/media_proxy.ts
@ -1,59 +1,71 @@
-import { NextApiRequest, NextApiResponse } from 'next'
-import redis from '../../utils/redis'
-import axiosInstance from '../../utils/axiosInstance'
-import { AxiosResponse } from 'axios'
+import { NextApiRequest, NextApiResponse } from 'next';
+import redis from '../../utils/redis';
+import axiosInstance from '../../utils/axiosInstance';

 const regex =
-  /^https:\/\/((m\.)?media-amazon\.com|imdb-video\.media-imdb\.com).*\.(jpg|jpeg|png|mp4|gif|webp).*$/
+  /^https:\/\/((m\.)?media-amazon\.com|imdb-video\.media-imdb\.com).*\.(jpg|jpeg|png|mp4|gif|webp).*$/;

 export default async function handler(
  req: NextApiRequest,
  res: NextApiResponse
 ) {
-  const mediaUrl = req.query.url as string | undefined
+  try {
+    const mediaUrl = req.query.url as string | undefined;

+    // 1. returning if query is illegal
    if (!mediaUrl || !regex.test(mediaUrl))
      return res.status(400).json({
        success: false,
        message: 'Invalid query',
-    })
+      });

-  if (process.env.USE_REDIS === 'true') {
-    const cachedMedia = await redis.getBuffer(mediaUrl)
+    // 2. sending streamed response if redis isn't enabled
+    if (redis === null) {
+      const mediaRes = await axiosInstance.get(mediaUrl, {
+        responseType: 'stream',
+      });
+
+      res.setHeader('Content-Type', mediaRes.headers['content-type']);
+      mediaRes.data.pipe(res);
+      return;
+    }
+
+    // 3. else if resourced is cached, sending it
+    const cachedMedia = await redis!.getBuffer(mediaUrl);

    if (cachedMedia) {
-      res.setHeader('x-cached', 'true')
-      res.status(302).send(cachedMedia)
-      return
-    }
+      res.setHeader('x-cached', 'true');
+      res.status(302).send(cachedMedia);
+      return;
    }

-  let mediaRes: AxiosResponse
-  try {
-    mediaRes = await axiosInstance(mediaUrl, { responseType: 'arraybuffer' })
+    // 4. else getting, caching and sending response
+    const mediaRes = await axiosInstance(mediaUrl, {
+      responseType: 'arraybuffer',
+    });
+
+    const data = mediaRes.data;
+
+    // saving in redis for 30 minutes
+    await redis!.setex(mediaUrl, 30 * 60, Buffer.from(data));
+
+    // sending media
+    res.setHeader('x-cached', 'false');
+    res.send(data);
+
+    // sending token response on any error
  } catch {
-    res.status(404)
+    res.status(404);
    res.json({
      success: false,
-      message: 'Error from IMDb',
-    })
-    return
+      message: 'something went wrong',
+    });
+    return;
  }
-
-  const data = mediaRes.data
-
-  if (process.env.USE_REDIS === 'true') {
-    // save in redis for 30 minutes
-    await redis.setex(mediaUrl, 30 * 60, Buffer.from(data))
-  }
-  
-  // send media
-  res.setHeader('x-cached', 'false')
-  res.send(data)
 }

 export const config = {
  api: {
    responseLimit: false,
  },
-}
+};
--- a/src/utils/redis.ts
+++ b/src/utils/redis.ts
@ -1,11 +1,11 @@
-import Redis from 'ioredis'
+import Redis from 'ioredis';

-const redisUrl = process.env.REDIS_URL
+const redisUrl = process.env.REDIS_URL;
+const toUseRedis = process.env.USE_REDIS === 'true';

-if (!redisUrl) {
-  throw 'Please set the REDIS_URL environment variable.'
-}
+let redis: Redis | null;

-const redis = new Redis(redisUrl)
+if (toUseRedis && redisUrl) redis = new Redis(redisUrl);
+else redis = null;

-export default redis
+export default redis;