feat(analytics): move the analytics report from api-gateway to analytics

packages/api-gateway/bin/report.ts → packages/analytics/bin/report.ts

@@ -4,30 +4,34 @@ import 'newrelic'
 
 import { Logger } from 'winston'
 
+import { DomainEventPublisherInterface } from '@standardnotes/domain-events'
+import { AnalyticsActivity } from '../src/Domain/Analytics/AnalyticsActivity'
+import { Period } from '../src/Domain/Time/Period'
+import { StatisticsMeasure } from '../src/Domain/Statistics/StatisticsMeasure'
+import { AnalyticsStoreInterface } from '../src/Domain/Analytics/AnalyticsStoreInterface'
+import { StatisticsStoreInterface } from '../src/Domain/Statistics/StatisticsStoreInterface'
+import { PeriodKeyGeneratorInterface } from '../src/Domain/Time/PeriodKeyGeneratorInterface'
 import { ContainerConfigLoader } from '../src/Bootstrap/Container'
 import TYPES from '../src/Bootstrap/Types'
 import { Env } from '../src/Bootstrap/Env'
-import {
-  DomainEventPublisherInterface,
-  DailyAnalyticsReportGeneratedEvent,
-  DomainEventService,
-} from '@standardnotes/domain-events'
-import {
-  AnalyticsActivity,
-  AnalyticsStoreInterface,
-  Period,
-  PeriodKeyGeneratorInterface,
-  StatisticsMeasure,
-  StatisticsStoreInterface,
-} from '@standardnotes/analytics'
+import { DomainEventFactoryInterface } from '../src/Domain/Event/DomainEventFactoryInterface'
 
 const requestReport = async (
   analyticsStore: AnalyticsStoreInterface,
   statisticsStore: StatisticsStoreInterface,
+  domainEventFactory: DomainEventFactoryInterface,
   domainEventPublisher: DomainEventPublisherInterface,
   periodKeyGenerator: PeriodKeyGeneratorInterface,
 ): Promise<void> => {
-  const analyticsOverTime = []
+  const analyticsOverTime: Array<{
+    name: string
+    period: number
+    counts: Array<{
+      periodKey: string
+      totalCount: number
+    }>
+    totalCount: number
+  }> = []
 
   const thirtyDaysAnalyticsNames = [
     AnalyticsActivity.GeneralActivity,
@@ -69,7 +73,11 @@ const requestReport = async (
     }
   }
 
-  const yesterdayActivityStatistics = []
+  const yesterdayActivityStatistics: Array<{
+    name: string
+    retention: number
+    totalCount: number
+  }> = []
   const yesterdayActivityNames = [
     AnalyticsActivity.LimitedDiscountOfferPurchased,
     AnalyticsActivity.GeneralActivity,
@@ -114,7 +122,13 @@ const requestReport = async (
     StatisticsMeasure.NewCustomers,
     StatisticsMeasure.TotalCustomers,
   ]
-  const statisticMeasures = []
+  const statisticMeasures: Array<{
+    name: string
+    totalValue: number
+    average: number
+    increments: number
+    period: number
+  }> = []
   for (const statisticMeasureName of statisticMeasureNames) {
     for (const period of [Period.Yesterday, Period.ThisMonth]) {
       statisticMeasures.push({
@@ -128,7 +142,11 @@ const requestReport = async (
   }
 
   const periodKeys = periodKeyGenerator.getDiscretePeriodKeys(Period.Last7Days)
-  const retentionOverDays = []
+  const retentionOverDays: Array<{
+    firstPeriodKey: string
+    secondPeriodKey: string
+    value: number
+  }> = []
   for (let i = 0; i < periodKeys.length; i++) {
     for (let j = 0; j < periodKeys.length - i; j++) {
       const dailyRetention = await analyticsStore.calculateActivitiesRetention({
@@ -147,7 +165,10 @@ const requestReport = async (
   }
 
   const monthlyPeriodKeys = periodKeyGenerator.getDiscretePeriodKeys(Period.ThisYear)
-  const churnRates = []
+  const churnRates: Array<{
+    rate: number
+    periodKey: string
+  }> = []
   for (const monthPeriodKey of monthlyPeriodKeys) {
     const monthPeriod = periodKeyGenerator.convertPeriodKeyToPeriod(monthPeriodKey)
     const dailyPeriodKeys = periodKeyGenerator.getDiscretePeriodKeys(monthPeriod)
@@ -179,39 +200,28 @@ const requestReport = async (
     })
   }
 
-  const event: DailyAnalyticsReportGeneratedEvent = {
-    type: 'DAILY_ANALYTICS_REPORT_GENERATED',
-    createdAt: new Date(),
-    meta: {
-      correlation: {
-        userIdentifier: '',
-        userIdentifierType: 'uuid',
-      },
-      origin: DomainEventService.ApiGateway,
-    },
-    payload: {
-      applicationStatistics: await statisticsStore.getYesterdayApplicationUsage(),
-      snjsStatistics: await statisticsStore.getYesterdaySNJSUsage(),
-      outOfSyncIncidents: await statisticsStore.getYesterdayOutOfSyncIncidents(),
-      activityStatistics: yesterdayActivityStatistics,
-      activityStatisticsOverTime: analyticsOverTime,
-      statisticMeasures,
-      retentionStatistics: [
-        {
-          firstActivity: AnalyticsActivity.Register,
-          secondActivity: AnalyticsActivity.GeneralActivity,
-          retention: {
-            periodKeys,
-            values: retentionOverDays,
-          },
+  const event = domainEventFactory.createDailyAnalyticsReportGeneratedEvent({
+    applicationStatistics: await statisticsStore.getYesterdayApplicationUsage(),
+    snjsStatistics: await statisticsStore.getYesterdaySNJSUsage(),
+    outOfSyncIncidents: await statisticsStore.getYesterdayOutOfSyncIncidents(),
+    activityStatistics: yesterdayActivityStatistics,
+    activityStatisticsOverTime: analyticsOverTime,
+    statisticMeasures,
+    retentionStatistics: [
+      {
+        firstActivity: AnalyticsActivity.Register,
+        secondActivity: AnalyticsActivity.GeneralActivity,
+        retention: {
+          periodKeys,
+          values: retentionOverDays,
         },
-      ],
-      churn: {
-        periodKeys: monthlyPeriodKeys,
-        values: churnRates,
       },
+    ],
+    churn: {
+      periodKeys: monthlyPeriodKeys,
+      values: churnRates,
     },
-  }
+  })
 
   await domainEventPublisher.publish(event)
 }
@@ -227,10 +237,13 @@ void container.load().then((container) => {
 
   const analyticsStore: AnalyticsStoreInterface = container.get(TYPES.AnalyticsStore)
   const statisticsStore: StatisticsStoreInterface = container.get(TYPES.StatisticsStore)
+  const domainEventFactory: DomainEventFactoryInterface = container.get(TYPES.DomainEventFactory)
   const domainEventPublisher: DomainEventPublisherInterface = container.get(TYPES.DomainEventPublisher)
   const periodKeyGenerator: PeriodKeyGeneratorInterface = container.get(TYPES.PeriodKeyGenerator)
 
-  Promise.resolve(requestReport(analyticsStore, statisticsStore, domainEventPublisher, periodKeyGenerator))
+  Promise.resolve(
+    requestReport(analyticsStore, statisticsStore, domainEventFactory, domainEventPublisher, periodKeyGenerator),
+  )
     .then(() => {
       logger.info('Usage report generation complete')
 

packages/analytics/docker/entrypoint.sh

@@ -9,6 +9,11 @@ case "$COMMAND" in
     yarn workspace @standardnotes/analytics worker
     ;;
 
+  'report' )
+    echo "Starting Usage Report Generation..."
+    yarn workspace @standardnotes/analytics report
+    ;;
+
    * )
     echo "Unknown command"
     ;;

packages/analytics/package.json

@@ -23,6 +23,7 @@
     "lint:fix": "eslint . --ext .ts --fix",
     "test": "jest --coverage --config=./jest.config.js --maxWorkers=50%",
     "worker": "yarn node dist/bin/worker.js",
+    "report": "yarn node dist/bin/report.js",
     "setup:env": "cp .env.sample .env",
     "typeorm": "typeorm-ts-node-commonjs"
   },

packages/analytics/src/Bootstrap/Container.ts

@@ -22,6 +22,12 @@ import {
   SQSNewRelicEventMessageHandler,
 } from '@standardnotes/domain-events-infra'
 import { Timer, TimerInterface } from '@standardnotes/time'
+import { PeriodKeyGeneratorInterface } from '../Domain/Time/PeriodKeyGeneratorInterface'
+import { PeriodKeyGenerator } from '../Domain/Time/PeriodKeyGenerator'
+import { AnalyticsStoreInterface } from '../Domain/Analytics/AnalyticsStoreInterface'
+import { RedisAnalyticsStore } from '../Infra/Redis/RedisAnalyticsStore'
+import { StatisticsStoreInterface } from '../Domain/Statistics/StatisticsStoreInterface'
+import { RedisStatisticsStore } from '../Infra/Redis/RedisStatisticsStore'
 
 // eslint-disable-next-line @typescript-eslint/no-var-requires
 const newrelicFormatter = require('@newrelic/winston-enricher')
@@ -100,6 +106,13 @@ export class ContainerConfigLoader {
 
     // Services
     container.bind<DomainEventFactory>(TYPES.DomainEventFactory).to(DomainEventFactory)
+    container.bind<PeriodKeyGeneratorInterface>(TYPES.PeriodKeyGenerator).toConstantValue(new PeriodKeyGenerator())
+    container
+      .bind<AnalyticsStoreInterface>(TYPES.AnalyticsStore)
+      .toConstantValue(new RedisAnalyticsStore(container.get(TYPES.PeriodKeyGenerator), container.get(TYPES.Redis)))
+    container
+      .bind<StatisticsStoreInterface>(TYPES.StatisticsStore)
+      .toConstantValue(new RedisStatisticsStore(container.get(TYPES.PeriodKeyGenerator), container.get(TYPES.Redis)))
     container.bind<TimerInterface>(TYPES.Timer).toConstantValue(new Timer())
 
     if (env.get('SNS_TOPIC_ARN', true)) {

packages/analytics/src/Bootstrap/Types.ts

@@ -20,7 +20,10 @@ const TYPES = {
   DomainEventSubscriberFactory: Symbol.for('DomainEventSubscriberFactory'),
   DomainEventFactory: Symbol.for('DomainEventFactory'),
   DomainEventMessageHandler: Symbol.for('DomainEventMessageHandler'),
+  AnalyticsStore: Symbol.for('AnalyticsStore'),
+  StatisticsStore: Symbol.for('StatisticsStore'),
   Timer: Symbol.for('Timer'),
+  PeriodKeyGenerator: Symbol.for('PeriodKeyGenerator'),
 }
 
 export default TYPES

packages/api-gateway/docker/entrypoint.sh

@@ -16,11 +16,6 @@ case "$COMMAND" in
     yarn workspace @standardnotes/api-gateway start
     ;;
 
-  'report' )
-    echo "Starting Usage Report Generation..."
-    yarn workspace @standardnotes/api-gateway report
-    ;;
-
    * )
     echo "Unknown command"
     ;;

packages/api-gateway/package.json

@@ -17,7 +17,6 @@
     "lint": "eslint . --ext .ts",
     "setup:env": "cp .env.sample .env",
     "start": "yarn node dist/bin/server.js",
-    "report": "yarn node dist/bin/report.js",
     "upgrade:snjs": "yarn ncu -u '@standardnotes/*'"
   },
   "dependencies": {