feat(cache): implement caching of routes

This commit is contained in:
zyachel 2023-05-21 18:15:03 +05:30
parent 8599ae2c5a
commit c53c88db9b
9 changed files with 82 additions and 42 deletions

View file

@ -24,10 +24,15 @@ NEXT_TELEMETRY_DISABLED=1
################################################################################
### 3. REDIS CONFIG(optional if you don't need redis)
################################################################################
## if you want to use redis to speed up the media proxy, set this to true
## enables caching of api routes as well as media
# USE_REDIS=true
## in case you don't want to cache media but only api routes
# USE_REDIS_FOR_API_ONLY=true
## ttl for media and api
# REDIS_CACHE_TTL_API=3600
# REDIS_CACHE_TTL_MEDIA=3600
## for docker, just set the domain to the container name, default is 'libremdb_redis'
REDIS_URL=localhost:6379
# REDIS_URL=localhost:6379
################################################################################
### 4. INSTANCE META FIELDS(not required but good to have)

View file

@ -112,7 +112,7 @@ Inspired by projects like [teddit](https://codeberg.org/teddit/teddit), [nitter]
- [ ] company info
- [ ] user info
- [ ] use redis, or any other caching strategy
- [X] use redis, or any other caching strategy
- [x] implement a better installation method
- [x] serve images and videos from libremdb itself

View file

@ -1,30 +1,32 @@
import { NextApiRequest, NextApiResponse } from 'next';
import { AxiosRequestHeaders } from 'axios';
import redis from 'src/utils/redis';
import axiosInstance from 'src/utils/axiosInstance';
import { mediaKey } from 'src/utils/constants/keys';
const getCleanReqHeaders = (headers: NextApiRequest['headers']) => ({
...(headers.accept && { accept: headers.accept }),
...(headers.range && { range: headers.range }),
...(headers['accept-encoding'] && {
'accept-encoding': headers['accept-encoding'] as string,
}),
});
const dontCacheMedia =
process.env.USE_REDIS_FOR_API_ONLY === 'true' || process.env.USE_REDIS !== 'true';
const resHeadersArr = [
'content-range',
'content-length',
'content-type',
'accept-ranges',
];
const ttl = process.env.REDIS_CACHE_TTL_MEDIA ?? 30 * 60;
const getCleanReqHeaders = (headers: NextApiRequest['headers']) => {
const cleanHeaders: AxiosRequestHeaders = {};
if (headers.accept) cleanHeaders.accept = headers.accept;
if (headers.range) cleanHeaders.range = headers.range;
if (headers['accept-encoding'])
cleanHeaders['accept-encoding'] = headers['accept-encoding'].toString();
return cleanHeaders;
};
const resHeadersArr = ['content-range', 'content-length', 'content-type', 'accept-ranges'];
// checks if a url is pointing towards a video/image from imdb
const regex =
/^https:\/\/((m\.)?media-amazon\.com|imdb-video\.media-imdb\.com).*\.(jpg|jpeg|png|mp4|gif|webp).*$/;
export default async function handler(
req: NextApiRequest,
res: NextApiResponse
) {
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
try {
const mediaUrl = req.query.url as string | undefined;
const requestHeaders = getCleanReqHeaders(req.headers);
@ -36,8 +38,8 @@ export default async function handler(
message: 'Invalid query',
});
// 2. sending streamed response if redis isn't enabled
if (redis === null) {
// 2. sending streamed response if redis, or redis for media isn't enabled
if (dontCacheMedia) {
const mediaRes = await axiosInstance.get(mediaUrl, {
responseType: 'stream',
headers: requestHeaders,
@ -54,23 +56,21 @@ export default async function handler(
}
// 3. else if resourced is cached, sending it
const cachedMedia = await redis!.getBuffer(mediaUrl);
const cachedMedia = await redis.getBuffer(mediaKey(mediaUrl));
if (cachedMedia) {
res.setHeader('x-cached', 'true');
res.status(302).send(cachedMedia);
res.status(304).send(cachedMedia);
return;
}
// 4. else getting, caching and sending response
const mediaRes = await axiosInstance(mediaUrl, {
const { data } = await axiosInstance(mediaUrl, {
responseType: 'arraybuffer',
});
const { data } = mediaRes;
// saving in redis for 30 minutes
await redis!.setex(mediaUrl, 30 * 60, Buffer.from(data));
await redis.setex(mediaKey(mediaUrl), ttl, Buffer.from(data));
// sending media
res.setHeader('x-cached', 'false');

View file

@ -1,4 +1,4 @@
import { GetServerSideProps } from 'next';
import { GetServerSideProps, InferGetServerSidePropsType } from 'next';
import Layout from 'src/layouts/Layout';
import ErrorInfo from 'src/components/error/ErrorInfo';
import Meta from 'src/components/meta/Meta';
@ -7,13 +7,12 @@ import Form from 'src/components/forms/find';
import Find, { FindQueryParams } from 'src/interfaces/shared/search';
import { AppError } from 'src/interfaces/shared/error';
import basicSearch from 'src/utils/fetchers/basicSearch';
import getOrSetApiCache from 'src/utils/getOrSetApiCache';
import { cleanQueryStr } from 'src/utils/helpers';
import { findKey } from 'src/utils/constants/keys';
import styles from 'src/styles/modules/pages/find/find.module.scss';
type Props =
| { data: { title: string; results: Find }; error: null }
| { data: { title: null; results: null }; error: null }
| { data: { title: string; results: null }; error: AppError };
type Props = InferGetServerSidePropsType<typeof getServerSideProps>;
const getMetadata = (title: string | null) => ({
title: title || 'Search',
@ -23,8 +22,7 @@ const getMetadata = (title: string | null) => ({
});
const BasicSearch = ({ data: { title, results }, error }: Props) => {
if (error)
return <ErrorInfo message={error.message} statusCode={error.statusCode} />;
if (error) return <ErrorInfo message={error.message} statusCode={error.statusCode} />;
return (
<>
@ -40,19 +38,23 @@ const BasicSearch = ({ data: { title, results }, error }: Props) => {
};
// TODO: use generics for passing in queryParams(to components) for better type-checking.
export const getServerSideProps: GetServerSideProps = async ctx => {
type Data =
| { data: { title: string; results: Find }; error: null }
| { data: { title: null; results: null }; error: null }
| { data: { title: string; results: null }; error: AppError };
export const getServerSideProps: GetServerSideProps<Data, FindQueryParams> = async ctx => {
// sample query str: find/?q=babylon&s=tt&ttype=ft&exact=true
const queryObj = ctx.query as FindQueryParams;
const query = queryObj.q?.trim();
if (!query)
return { props: { data: { title: null, results: null }, error: null } };
if (!query) return { props: { data: { title: null, results: null }, error: null } };
try {
const entries = Object.entries(queryObj);
const queryStr = cleanQueryStr(entries);
const res = await basicSearch(queryStr);
const res = await getOrSetApiCache(findKey(queryStr), basicSearch, queryStr);
return {
props: { data: { title: query, results: res }, error: null },

View file

@ -7,7 +7,9 @@ import { Basic, Credits, DidYouKnow, Info, Bio, KnownFor } from 'src/components/
import Name from 'src/interfaces/shared/name';
import { AppError } from 'src/interfaces/shared/error';
import name from 'src/utils/fetchers/name';
import getOrSetApiCache from 'src/utils/getOrSetApiCache';
import { getProxiedIMDbImgUrl } from 'src/utils/helpers';
import { nameKey } from 'src/utils/constants/keys';
import styles from 'src/styles/modules/pages/name/name.module.scss';
type Props = InferGetServerSidePropsType<typeof getServerSideProps>;
@ -46,7 +48,7 @@ export const getServerSideProps: GetServerSideProps<Data, Params> = async ctx =>
const nameId = ctx.params!.nameId;
try {
const data = await name(nameId);
const data = await getOrSetApiCache(nameKey(nameId), name, nameId);
return { props: { data, error: null } };
} catch (error: any) {

View file

@ -6,8 +6,10 @@ import Media from 'src/components/media/Media';
import { Basic, Cast, DidYouKnow, Info, MoreLikeThis, Reviews } from 'src/components/title';
import Title from 'src/interfaces/shared/title';
import { AppError } from 'src/interfaces/shared/error';
import getOrSetApiCache from 'src/utils/getOrSetApiCache';
import title from 'src/utils/fetchers/title';
import { getProxiedIMDbImgUrl } from 'src/utils/helpers';
import { titleKey } from 'src/utils/constants/keys';
import styles from 'src/styles/modules/pages/title/title.module.scss';
type Props = InferGetServerSidePropsType<typeof getServerSideProps>;
@ -55,7 +57,7 @@ export const getServerSideProps: GetServerSideProps<Data, Params> = async ctx =>
const titleId = ctx.params!.titleId;
try {
const data = await title(titleId);
const data = await getOrSetApiCache(titleKey(titleId), title, titleId);
return { props: { data, error: null } };
} catch (error: any) {

View file

@ -0,0 +1,4 @@
export const titleKey = (titleId: string) => `title:${titleId}`;
export const nameKey = (nameId: string) => `name:${nameId}`;
export const findKey = (query: string) => `find:${query}`;
export const mediaKey = (url: string) => `media:${url}`;

View file

@ -0,0 +1,24 @@
import redis from 'src/utils/redis';
const ttl = process.env.REDIS_CACHE_TTL_API ?? 30 * 60;
const redisEnabled =
process.env.USE_REDIS === 'true' || process.env.USE_REDIS_FOR_API_ONLY === 'true';
const getOrSetApiCache = async <T extends (...fetcherArgs: any[]) => Promise<any>>(
key: string,
fetcher: T,
...fetcherArgs: Parameters<T>
): Promise<ReturnType<T>> => {
if (!redisEnabled) return await fetcher(...fetcherArgs);
const dataInCache = await redis.get(key);
if (dataInCache) return JSON.parse(dataInCache);
const dataToCache = await fetcher(...fetcherArgs);
await redis.setex(key, ttl, JSON.stringify(dataToCache));
return dataToCache;
};
export default getOrSetApiCache;

View file

@ -2,7 +2,8 @@
import Redis from 'ioredis';
const redisUrl = process.env.REDIS_URL;
const toUseRedis = process.env.USE_REDIS === 'true';
const toUseRedis =
process.env.USE_REDIS === 'true' || process.env.USE_REDIS_FOR_API_ONLY === 'true';
const stub: Pick<Redis, 'get' | 'setex' | 'getBuffer'> = {
get: async key => Promise.resolve(null),