From c205ebb7101705572d8be96173b02870893a2630 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karol=20S=C3=B3jko?= Date: Tue, 21 Jun 2022 13:53:26 +0200 Subject: [PATCH] fix: docker image building --- .github/workflows/scheduler.release.dev.yml | 30 ++++++++----------- .github/workflows/version-bump.yml | 2 +- .../plugins/@yarnpkg/plugin-docker-build.cjs | 8 +++++ .yarnrc.yml | 2 ++ Dockerfile.scheduler | 28 ----------------- packages/scheduler/Dockerfile | 23 ++++++++++++++ packages/scheduler/package.json | 2 +- yarn.lock | 4 +-- 8 files changed, 50 insertions(+), 49 deletions(-) create mode 100644 .yarn/plugins/@yarnpkg/plugin-docker-build.cjs delete mode 100644 Dockerfile.scheduler create mode 100644 packages/scheduler/Dockerfile diff --git a/.github/workflows/scheduler.release.dev.yml b/.github/workflows/scheduler.release.dev.yml index 869b8075d..238fca1fd 100644 --- a/.github/workflows/scheduler.release.dev.yml +++ b/.github/workflows/scheduler.release.dev.yml @@ -20,7 +20,6 @@ jobs: - uses: actions/setup-node@v1 with: node-version: '16.x' - - run: yarn install - run: yarn test:scheduler publish-aws-ecr: @@ -39,10 +38,6 @@ jobs: - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v1 - - name: Install dependencies locally - run: yarn install - - name: Build application locally - run: yarn build:scheduler - name: Build, tag, and push image to Amazon ECR id: build-image env: @@ -50,7 +45,7 @@ jobs: ECR_REPOSITORY: scheduler-worker IMAGE_TAG: ${{ github.sha }} run: | - docker build -f Dockerfile.scheduler -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + yarn docker build @standardnotes/scheduler-server -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:dev docker push $ECR_REGISTRY/$ECR_REPOSITORY:dev @@ -62,21 +57,22 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Install dependencies locally - run: yarn install - - name: Build application locally - run: yarn build:scheduler - - name: Publish to Registry - env: - DOCKER_BUILDKIT: 1 - COMPOSE_DOCKER_CLI_BUILD: 1 - uses: elgohr/Publish-Docker-Github-Action@master + - name: Build image + run: yarn docker build @standardnotes/scheduler-server -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + - name: Publish to Registry (dev tag) + uses: manusa/actions-publish-docker@v1.1.2 with: name: standardnotes/scheduler username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - dockerfile: Dockerfile.scheduler - tags: "dev,${{ github.sha }}" + tag: "dev" + - name: Publish to Registry (sha tag) + uses: manusa/actions-publish-docker@v1.1.2 + with: + name: standardnotes/scheduler + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + tag: "${{ github.sha }}" deploy-worker: needs: publish-aws-ecr diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 171a519f4..0b7c0ed87 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -32,7 +32,7 @@ jobs: with: node-version: '16.x' - - run: yarn install + - run: yarn install --immutable - name: Bump Prod Version if: ${{ github.ref == 'refs/heads/main' }} diff --git a/.yarn/plugins/@yarnpkg/plugin-docker-build.cjs b/.yarn/plugins/@yarnpkg/plugin-docker-build.cjs new file mode 100644 index 000000000..a1e572658 --- /dev/null +++ b/.yarn/plugins/@yarnpkg/plugin-docker-build.cjs @@ -0,0 +1,8 @@ +/* eslint-disable */ +module.exports = { +name: "@yarnpkg/plugin-docker-build", +factory: function (require) { +var plugin;(()=>{"use strict";var t={d:(e,o)=>{for(var r in o)t.o(o,r)&&!t.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:o[r]})},o:(t,e)=>Object.prototype.hasOwnProperty.call(t,e),r:t=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(t,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(t,"__esModule",{value:!0})}},e={};t.r(e),t.d(e,{default:()=>u});const o=require("@yarnpkg/cli"),r=require("clipanion"),i=require("@yarnpkg/core"),a=require("@yarnpkg/plugin-patch"),n=require("@yarnpkg/fslib");const s=require("@yarnpkg/plugin-pack");async function c({workspace:t,destination:e,report:o}){await s.packUtils.prepareForPack(t,{report:o},async()=>{const r=await s.packUtils.genPackList(t),a=i.Report.progressViaCounter(r.length),c=o.reportProgress(a);try{for(const i of r){const r=n.ppath.join(t.cwd,i),s=n.ppath.join(e,t.relativeCwd,i);o.reportInfo(null,i),await n.xfs.copyPromise(s,r,{overwrite:!0}),a.tick()}}finally{c.stop()}})}function p(t,e){const o=(0,n.toFilename)(e);return n.ppath.isAbsolute(o)?n.ppath.relative(t,o):o}const l=/^builtin<([^>]+)>$/;var d=function(t,e,o,r){var i,a=arguments.length,n=a<3?e:null===r?r=Object.getOwnPropertyDescriptor(e,o):r;if("object"==typeof Reflect&&"function"==typeof Reflect.decorate)n=Reflect.decorate(t,e,o,r);else for(var s=t.length-1;s>=0;s--)(i=t[s])&&(n=(a<3?i(n):a>3?i(e,o,n):i(e,o))||n);return a>3&&n&&Object.defineProperty(e,o,n),n};class f extends o.BaseCommand{constructor(){super(...arguments),this.args=[]}async execute(){const t=await i.Configuration.find(this.context.cwd,this.context.plugins),{project:e}=await i.Project.find(t,this.context.cwd),o=e.getWorkspaceByIdent(i.structUtils.parseIdent(this.workspaceName)),r=function({project:t,workspaces:e,production:o=!1,scopes:r=(o?["dependencies"]:i.Manifest.hardDependencies)}){const a=new Set([...e]);for(const e of a)for(const o of r){const r=e.manifest.getForScope(o).values();for(const e of r){const o=t.tryWorkspaceByDescriptor(e);o&&a.add(o)}}for(const e of t.workspaces)a.has(e)?o&&e.manifest.devDependencies.clear():(e.manifest.dependencies.clear(),e.manifest.devDependencies.clear(),e.manifest.peerDependencies.clear());return a}({project:e,workspaces:[o],production:this.production}),s=await async function(t,e="Dockerfile"){const o=(0,n.toFilename)(e);if(n.ppath.isAbsolute(o))return o;const r=[n.ppath.join(t.cwd,o),n.ppath.join(t.project.cwd,o)];for(const t of r)if(await n.xfs.existsPromise(t))return t;throw new Error("Dockerfile is required")}(o,this.dockerFilePath),d=await i.Cache.find(t);return(await i.StreamReport.start({configuration:t,stdout:this.context.stdout,includeLogs:!this.context.quiet},async t=>{await t.startTimerPromise("Resolution Step",async()=>{await e.resolveEverything({report:t,cache:d})}),await t.startTimerPromise("Fetch Step",async()=>{await e.fetchEverything({report:t,cache:d})}),await n.xfs.mktempPromise(async o=>{const f=n.ppath.join(o,(0,n.toFilename)("manifests")),u=n.ppath.join(o,(0,n.toFilename)("packs"));await t.startTimerPromise("Copy files",async()=>{await async function({destination:t,project:e,report:o}){const r=e.configuration.get("rcFilename");o.reportInfo(null,r),await n.xfs.copyPromise(n.ppath.join(t,r),n.ppath.join(e.cwd,r),{overwrite:!0})}({destination:f,project:e,report:t}),await async function({destination:t,project:e,report:o}){const r=n.ppath.join((0,n.toFilename)(".yarn"),(0,n.toFilename)("plugins"));o.reportInfo(null,r),await n.xfs.copyPromise(n.ppath.join(t,r),n.ppath.join(e.cwd,r),{overwrite:!0})}({destination:f,project:e,report:t}),await async function({destination:t,project:e,report:o}){const r=e.configuration.get("yarnPath"),i=n.ppath.relative(e.cwd,r),a=n.ppath.join(t,i);o.reportInfo(null,i),await n.xfs.copyPromise(a,r,{overwrite:!0})}({destination:f,project:e,report:t}),await async function({destination:t,workspaces:e,report:o}){for(const r of e){const e=n.ppath.join(r.relativeCwd,i.Manifest.fileName),a=n.ppath.join(t,e),s={};r.manifest.exportTo(s),o.reportInfo(null,e),await n.xfs.mkdirpPromise(n.ppath.dirname(a)),await n.xfs.writeJsonPromise(a,s)}}({destination:f,workspaces:e.workspaces,report:t}),await async function({destination:t,report:e,project:o,parseDescriptor:r}){const a=new Set;for(const s of o.storedDescriptors.values()){const c=r(i.structUtils.isVirtualDescriptor(s)?i.structUtils.devirtualizeDescriptor(s):s);if(!c)continue;const{parentLocator:p,paths:d}=c;for(const r of d){if(l.test(r))continue;if(n.ppath.isAbsolute(r))continue;const i=o.getWorkspaceByLocator(p),s=n.ppath.join(i.relativeCwd,r);if(a.has(s))continue;a.add(s);const c=n.ppath.join(i.cwd,r),d=n.ppath.join(t,s);e.reportInfo(null,s),await n.xfs.mkdirpPromise(n.ppath.dirname(d)),await n.xfs.copyFilePromise(c,d)}}}({destination:f,report:t,project:e,parseDescriptor:t=>{if(t.range.startsWith("exec:")){const e=function(t){const{params:e,selector:o}=i.structUtils.parseRange(t),r=n.npath.toPortablePath(o);return{parentLocator:e&&"string"==typeof e.locator?i.structUtils.parseLocator(e.locator):null,path:r}}(t.range);if(!e||!e.parentLocator)return;return{parentLocator:e.parentLocator,paths:[e.path]}}if(t.range.startsWith("patch:")){const{parentLocator:e,patchPaths:o}=a.patchUtils.parseDescriptor(t);if(!e)return;return{parentLocator:e,paths:o}}}}),await async function({destination:t,project:e,cache:o,report:r}){for(const i of o.markedFiles){const o=n.ppath.relative(e.cwd,i);await n.xfs.existsPromise(i)&&(r.reportInfo(null,o),await n.xfs.copyPromise(n.ppath.join(t,o),i))}}({destination:f,project:e,cache:d,report:t}),await async function({destination:t,project:e,report:o}){const r=(0,n.toFilename)(e.configuration.get("lockfileFilename")),i=n.ppath.join(t,r);o.reportInfo(null,r),await n.xfs.mkdirpPromise(n.ppath.dirname(i)),await n.xfs.writeFilePromise(i,e.generateLockfile())}({destination:f,project:e,report:t}),this.copyFiles&&this.copyFiles.length&&await async function({destination:t,files:e,dockerFilePath:o,report:r}){const i=n.ppath.dirname(o);for(const o of e){const e=p(i,o),a=n.ppath.join(i,e),s=n.ppath.join(t,e);r.reportInfo(null,e),await n.xfs.copyPromise(s,a)}}({destination:f,files:this.copyFiles,dockerFilePath:s,report:t})});for(const e of r){const o=e.manifest.name?i.structUtils.stringifyIdent(e.manifest.name):"";await t.startTimerPromise("Pack workspace "+o,async()=>{await c({workspace:e,report:t,destination:u})})}await i.execUtils.pipevp("docker",["build",...this.args,"-f",s,"."],{cwd:o,strict:!0,stdin:this.context.stdin,stdout:this.context.stdout,stderr:this.context.stderr})})})).exitCode()}}f.usage=r.Command.Usage({category:"Docker-related commands",description:"Build a Docker image for a workspace",details:'\n This command will build a efficient Docker image which only contains necessary dependencies for the specified workspace.\n\n You have to create a Dockerfile in your workspace or your project. You can also specify the path to Dockerfile using the "-f, --file" option.\n\n Additional arguments can be passed to "docker build" directly, please check the Docker docs for more info: https://docs.docker.com/engine/reference/commandline/build/\n\n You can copy additional files or folders to a Docker image using the "--copy" option. This is useful for secret keys or configuration files. The files will be copied to "manifests" folder. The path can be either a path relative to the Dockerfile or an absolute path.\n ',examples:[["Build a Docker image for a workspace","yarn docker build @foo/bar"],["Pass additional arguments to docker build command","yarn docker build @foo/bar -t image-tag"],["Copy additional files to a Docker image","yarn docker build --copy secret.key --copy config.json @foo/bar"],["Install production dependencies only","yarn docker build --production @foo/bar"]]}),d([r.Command.String()],f.prototype,"workspaceName",void 0),d([r.Command.Proxy()],f.prototype,"args",void 0),d([r.Command.String("-f,--file")],f.prototype,"dockerFilePath",void 0),d([r.Command.Array("--copy")],f.prototype,"copyFiles",void 0),d([r.Command.Boolean("--production")],f.prototype,"production",void 0),d([r.Command.Path("docker","build")],f.prototype,"execute",null);const u={commands:[f]};plugin=e})(); +return plugin; +} +}; \ No newline at end of file diff --git a/.yarnrc.yml b/.yarnrc.yml index 2d58ad80c..bdd851b39 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -1,5 +1,7 @@ plugins: - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs spec: "@yarnpkg/plugin-workspace-tools" + - path: .yarn/plugins/@yarnpkg/plugin-docker-build.cjs + spec: "https://github.com/Dcard/yarn-plugins/releases/latest/download/plugin-docker-build.js" yarnPath: .yarn/releases/yarn-3.2.1.cjs diff --git a/Dockerfile.scheduler b/Dockerfile.scheduler deleted file mode 100644 index 9714d9147..000000000 --- a/Dockerfile.scheduler +++ /dev/null @@ -1,28 +0,0 @@ -FROM node:16.13.1-alpine3.15 - -ARG UID=1001 -ARG GID=1001 - -RUN addgroup -S scheduler -g $GID && adduser -D -S scheduler -G scheduler -u $UID - -RUN apk add --update --no-cache \ - curl \ - alpine-sdk \ - python3 - -WORKDIR /var/www - -RUN chown -R $UID:$GID . - -USER scheduler - -COPY --chown=$UID:$GID packages/scheduler/package.json .yarnrc.yml /var/www/ -COPY --chown=$UID:$GID .yarn /var/www/.yarn -COPY --chown=$UID:$GID packages/scheduler/dist /var/www/dist -COPY --chown=$UID:$GID packages/scheduler/docker /var/www/docker - -RUN yarn install - -ENTRYPOINT [ "docker/entrypoint.sh" ] - -CMD [ "start-worker" ] diff --git a/packages/scheduler/Dockerfile b/packages/scheduler/Dockerfile new file mode 100644 index 000000000..4a67e5933 --- /dev/null +++ b/packages/scheduler/Dockerfile @@ -0,0 +1,23 @@ +FROM node:16.13.1-alpine AS builder + +# Install dependencies for building native libraries +RUN apk add --update git openssh-client python3 alpine-sdk + +WORKDIR /workspace + +# docker-build plugin copies everything needed for `yarn install` to `manifests` folder. +COPY manifests ./ + +RUN yarn install --immutable + +FROM node:16.13.1-alpine + +WORKDIR /workspace + +# Copy the installed dependencies from the previous stage. +COPY --from=builder /workspace ./ + +# docker-build plugin runs `yarn pack` in all workspace dependencies and copies them to `packs` folder. +COPY packs ./ + +CMD yarn workspace @standardnotes/scheduler-server worker diff --git a/packages/scheduler/package.json b/packages/scheduler/package.json index 492ce84aa..a0c155acd 100644 --- a/packages/scheduler/package.json +++ b/packages/scheduler/package.json @@ -36,7 +36,7 @@ "ioredis": "^5.0.6", "mysql2": "^2.3.3", "newrelic": "8.6.0", - "reflect-metadata": "0.1.13", + "reflect-metadata": "^0.1.13", "typeorm": "^0.3.6", "winston": "3.3.3" }, diff --git a/yarn.lock b/yarn.lock index 62fc4e909..93e5ff139 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1757,7 +1757,7 @@ __metadata: ioredis: ^5.0.6 mysql2: ^2.3.3 newrelic: 8.6.0 - reflect-metadata: 0.1.13 + reflect-metadata: ^0.1.13 typeorm: ^0.3.6 winston: 3.3.3 languageName: unknown @@ -7151,7 +7151,7 @@ __metadata: languageName: node linkType: hard -"reflect-metadata@npm:0.1.13, reflect-metadata@npm:^0.1.13": +"reflect-metadata@npm:^0.1.13": version: 0.1.13 resolution: "reflect-metadata@npm:0.1.13" checksum: 798d379a7b6f6455501145419505c97dd11cbc23857a386add2b9ef15963ccf15a48d9d15507afe01d4cd74116df8a213247200bac00320bd7c11ddeaa5e8fb4