feat(cache): implement caching of routes
This commit is contained in:
parent
8599ae2c5a
commit
c53c88db9b
9 changed files with 82 additions and 42 deletions
|
@ -24,10 +24,15 @@ NEXT_TELEMETRY_DISABLED=1
|
|||
################################################################################
|
||||
### 3. REDIS CONFIG(optional if you don't need redis)
|
||||
################################################################################
|
||||
## if you want to use redis to speed up the media proxy, set this to true
|
||||
## enables caching of api routes as well as media
|
||||
# USE_REDIS=true
|
||||
## in case you don't want to cache media but only api routes
|
||||
# USE_REDIS_FOR_API_ONLY=true
|
||||
## ttl for media and api
|
||||
# REDIS_CACHE_TTL_API=3600
|
||||
# REDIS_CACHE_TTL_MEDIA=3600
|
||||
## for docker, just set the domain to the container name, default is 'libremdb_redis'
|
||||
REDIS_URL=localhost:6379
|
||||
# REDIS_URL=localhost:6379
|
||||
|
||||
################################################################################
|
||||
### 4. INSTANCE META FIELDS(not required but good to have)
|
||||
|
|
|
@ -112,7 +112,7 @@ Inspired by projects like [teddit](https://codeberg.org/teddit/teddit), [nitter]
|
|||
- [ ] company info
|
||||
- [ ] user info
|
||||
|
||||
- [ ] use redis, or any other caching strategy
|
||||
- [X] use redis, or any other caching strategy
|
||||
- [x] implement a better installation method
|
||||
- [x] serve images and videos from libremdb itself
|
||||
|
||||
|
|
|
@ -1,30 +1,32 @@
|
|||
import { NextApiRequest, NextApiResponse } from 'next';
|
||||
import { AxiosRequestHeaders } from 'axios';
|
||||
import redis from 'src/utils/redis';
|
||||
import axiosInstance from 'src/utils/axiosInstance';
|
||||
import { mediaKey } from 'src/utils/constants/keys';
|
||||
|
||||
const getCleanReqHeaders = (headers: NextApiRequest['headers']) => ({
|
||||
...(headers.accept && { accept: headers.accept }),
|
||||
...(headers.range && { range: headers.range }),
|
||||
...(headers['accept-encoding'] && {
|
||||
'accept-encoding': headers['accept-encoding'] as string,
|
||||
}),
|
||||
});
|
||||
const dontCacheMedia =
|
||||
process.env.USE_REDIS_FOR_API_ONLY === 'true' || process.env.USE_REDIS !== 'true';
|
||||
|
||||
const resHeadersArr = [
|
||||
'content-range',
|
||||
'content-length',
|
||||
'content-type',
|
||||
'accept-ranges',
|
||||
];
|
||||
const ttl = process.env.REDIS_CACHE_TTL_MEDIA ?? 30 * 60;
|
||||
|
||||
const getCleanReqHeaders = (headers: NextApiRequest['headers']) => {
|
||||
const cleanHeaders: AxiosRequestHeaders = {};
|
||||
|
||||
if (headers.accept) cleanHeaders.accept = headers.accept;
|
||||
if (headers.range) cleanHeaders.range = headers.range;
|
||||
if (headers['accept-encoding'])
|
||||
cleanHeaders['accept-encoding'] = headers['accept-encoding'].toString();
|
||||
|
||||
return cleanHeaders;
|
||||
};
|
||||
|
||||
const resHeadersArr = ['content-range', 'content-length', 'content-type', 'accept-ranges'];
|
||||
|
||||
// checks if a url is pointing towards a video/image from imdb
|
||||
const regex =
|
||||
/^https:\/\/((m\.)?media-amazon\.com|imdb-video\.media-imdb\.com).*\.(jpg|jpeg|png|mp4|gif|webp).*$/;
|
||||
|
||||
export default async function handler(
|
||||
req: NextApiRequest,
|
||||
res: NextApiResponse
|
||||
) {
|
||||
export default async function handler(req: NextApiRequest, res: NextApiResponse) {
|
||||
try {
|
||||
const mediaUrl = req.query.url as string | undefined;
|
||||
const requestHeaders = getCleanReqHeaders(req.headers);
|
||||
|
@ -36,8 +38,8 @@ export default async function handler(
|
|||
message: 'Invalid query',
|
||||
});
|
||||
|
||||
// 2. sending streamed response if redis isn't enabled
|
||||
if (redis === null) {
|
||||
// 2. sending streamed response if redis, or redis for media isn't enabled
|
||||
if (dontCacheMedia) {
|
||||
const mediaRes = await axiosInstance.get(mediaUrl, {
|
||||
responseType: 'stream',
|
||||
headers: requestHeaders,
|
||||
|
@ -54,23 +56,21 @@ export default async function handler(
|
|||
}
|
||||
|
||||
// 3. else if resourced is cached, sending it
|
||||
const cachedMedia = await redis!.getBuffer(mediaUrl);
|
||||
const cachedMedia = await redis.getBuffer(mediaKey(mediaUrl));
|
||||
|
||||
if (cachedMedia) {
|
||||
res.setHeader('x-cached', 'true');
|
||||
res.status(302).send(cachedMedia);
|
||||
res.status(304).send(cachedMedia);
|
||||
return;
|
||||
}
|
||||
|
||||
// 4. else getting, caching and sending response
|
||||
const mediaRes = await axiosInstance(mediaUrl, {
|
||||
const { data } = await axiosInstance(mediaUrl, {
|
||||
responseType: 'arraybuffer',
|
||||
});
|
||||
|
||||
const { data } = mediaRes;
|
||||
|
||||
// saving in redis for 30 minutes
|
||||
await redis!.setex(mediaUrl, 30 * 60, Buffer.from(data));
|
||||
await redis.setex(mediaKey(mediaUrl), ttl, Buffer.from(data));
|
||||
|
||||
// sending media
|
||||
res.setHeader('x-cached', 'false');
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { GetServerSideProps } from 'next';
|
||||
import { GetServerSideProps, InferGetServerSidePropsType } from 'next';
|
||||
import Layout from 'src/layouts/Layout';
|
||||
import ErrorInfo from 'src/components/error/ErrorInfo';
|
||||
import Meta from 'src/components/meta/Meta';
|
||||
|
@ -7,13 +7,12 @@ import Form from 'src/components/forms/find';
|
|||
import Find, { FindQueryParams } from 'src/interfaces/shared/search';
|
||||
import { AppError } from 'src/interfaces/shared/error';
|
||||
import basicSearch from 'src/utils/fetchers/basicSearch';
|
||||
import getOrSetApiCache from 'src/utils/getOrSetApiCache';
|
||||
import { cleanQueryStr } from 'src/utils/helpers';
|
||||
import { findKey } from 'src/utils/constants/keys';
|
||||
import styles from 'src/styles/modules/pages/find/find.module.scss';
|
||||
|
||||
type Props =
|
||||
| { data: { title: string; results: Find }; error: null }
|
||||
| { data: { title: null; results: null }; error: null }
|
||||
| { data: { title: string; results: null }; error: AppError };
|
||||
type Props = InferGetServerSidePropsType<typeof getServerSideProps>;
|
||||
|
||||
const getMetadata = (title: string | null) => ({
|
||||
title: title || 'Search',
|
||||
|
@ -23,8 +22,7 @@ const getMetadata = (title: string | null) => ({
|
|||
});
|
||||
|
||||
const BasicSearch = ({ data: { title, results }, error }: Props) => {
|
||||
if (error)
|
||||
return <ErrorInfo message={error.message} statusCode={error.statusCode} />;
|
||||
if (error) return <ErrorInfo message={error.message} statusCode={error.statusCode} />;
|
||||
|
||||
return (
|
||||
<>
|
||||
|
@ -40,19 +38,23 @@ const BasicSearch = ({ data: { title, results }, error }: Props) => {
|
|||
};
|
||||
|
||||
// TODO: use generics for passing in queryParams(to components) for better type-checking.
|
||||
export const getServerSideProps: GetServerSideProps = async ctx => {
|
||||
type Data =
|
||||
| { data: { title: string; results: Find }; error: null }
|
||||
| { data: { title: null; results: null }; error: null }
|
||||
| { data: { title: string; results: null }; error: AppError };
|
||||
|
||||
export const getServerSideProps: GetServerSideProps<Data, FindQueryParams> = async ctx => {
|
||||
// sample query str: find/?q=babylon&s=tt&ttype=ft&exact=true
|
||||
const queryObj = ctx.query as FindQueryParams;
|
||||
const query = queryObj.q?.trim();
|
||||
|
||||
if (!query)
|
||||
return { props: { data: { title: null, results: null }, error: null } };
|
||||
if (!query) return { props: { data: { title: null, results: null }, error: null } };
|
||||
|
||||
try {
|
||||
const entries = Object.entries(queryObj);
|
||||
const queryStr = cleanQueryStr(entries);
|
||||
|
||||
const res = await basicSearch(queryStr);
|
||||
const res = await getOrSetApiCache(findKey(queryStr), basicSearch, queryStr);
|
||||
|
||||
return {
|
||||
props: { data: { title: query, results: res }, error: null },
|
||||
|
|
|
@ -7,7 +7,9 @@ import { Basic, Credits, DidYouKnow, Info, Bio, KnownFor } from 'src/components/
|
|||
import Name from 'src/interfaces/shared/name';
|
||||
import { AppError } from 'src/interfaces/shared/error';
|
||||
import name from 'src/utils/fetchers/name';
|
||||
import getOrSetApiCache from 'src/utils/getOrSetApiCache';
|
||||
import { getProxiedIMDbImgUrl } from 'src/utils/helpers';
|
||||
import { nameKey } from 'src/utils/constants/keys';
|
||||
import styles from 'src/styles/modules/pages/name/name.module.scss';
|
||||
|
||||
type Props = InferGetServerSidePropsType<typeof getServerSideProps>;
|
||||
|
@ -46,7 +48,7 @@ export const getServerSideProps: GetServerSideProps<Data, Params> = async ctx =>
|
|||
const nameId = ctx.params!.nameId;
|
||||
|
||||
try {
|
||||
const data = await name(nameId);
|
||||
const data = await getOrSetApiCache(nameKey(nameId), name, nameId);
|
||||
|
||||
return { props: { data, error: null } };
|
||||
} catch (error: any) {
|
||||
|
|
|
@ -6,8 +6,10 @@ import Media from 'src/components/media/Media';
|
|||
import { Basic, Cast, DidYouKnow, Info, MoreLikeThis, Reviews } from 'src/components/title';
|
||||
import Title from 'src/interfaces/shared/title';
|
||||
import { AppError } from 'src/interfaces/shared/error';
|
||||
import getOrSetApiCache from 'src/utils/getOrSetApiCache';
|
||||
import title from 'src/utils/fetchers/title';
|
||||
import { getProxiedIMDbImgUrl } from 'src/utils/helpers';
|
||||
import { titleKey } from 'src/utils/constants/keys';
|
||||
import styles from 'src/styles/modules/pages/title/title.module.scss';
|
||||
|
||||
type Props = InferGetServerSidePropsType<typeof getServerSideProps>;
|
||||
|
@ -55,7 +57,7 @@ export const getServerSideProps: GetServerSideProps<Data, Params> = async ctx =>
|
|||
const titleId = ctx.params!.titleId;
|
||||
|
||||
try {
|
||||
const data = await title(titleId);
|
||||
const data = await getOrSetApiCache(titleKey(titleId), title, titleId);
|
||||
|
||||
return { props: { data, error: null } };
|
||||
} catch (error: any) {
|
||||
|
|
4
src/utils/constants/keys.ts
Normal file
4
src/utils/constants/keys.ts
Normal file
|
@ -0,0 +1,4 @@
|
|||
export const titleKey = (titleId: string) => `title:${titleId}`;
|
||||
export const nameKey = (nameId: string) => `name:${nameId}`;
|
||||
export const findKey = (query: string) => `find:${query}`;
|
||||
export const mediaKey = (url: string) => `media:${url}`;
|
24
src/utils/getOrSetApiCache.ts
Normal file
24
src/utils/getOrSetApiCache.ts
Normal file
|
@ -0,0 +1,24 @@
|
|||
import redis from 'src/utils/redis';
|
||||
|
||||
const ttl = process.env.REDIS_CACHE_TTL_API ?? 30 * 60;
|
||||
const redisEnabled =
|
||||
process.env.USE_REDIS === 'true' || process.env.USE_REDIS_FOR_API_ONLY === 'true';
|
||||
|
||||
const getOrSetApiCache = async <T extends (...fetcherArgs: any[]) => Promise<any>>(
|
||||
key: string,
|
||||
fetcher: T,
|
||||
...fetcherArgs: Parameters<T>
|
||||
): Promise<ReturnType<T>> => {
|
||||
if (!redisEnabled) return await fetcher(...fetcherArgs);
|
||||
|
||||
const dataInCache = await redis.get(key);
|
||||
if (dataInCache) return JSON.parse(dataInCache);
|
||||
|
||||
const dataToCache = await fetcher(...fetcherArgs);
|
||||
|
||||
await redis.setex(key, ttl, JSON.stringify(dataToCache));
|
||||
|
||||
return dataToCache;
|
||||
};
|
||||
|
||||
export default getOrSetApiCache;
|
|
@ -2,7 +2,8 @@
|
|||
import Redis from 'ioredis';
|
||||
|
||||
const redisUrl = process.env.REDIS_URL;
|
||||
const toUseRedis = process.env.USE_REDIS === 'true';
|
||||
const toUseRedis =
|
||||
process.env.USE_REDIS === 'true' || process.env.USE_REDIS_FOR_API_ONLY === 'true';
|
||||
|
||||
const stub: Pick<Redis, 'get' | 'setex' | 'getBuffer'> = {
|
||||
get: async key => Promise.resolve(null),
|
||||
|
|
Loading…
Reference in a new issue