(feature) Predefined funnels (#199)

* added an ability to store prefedined funnels into the database

* ADD GET all funnels

* added some account validations

* added funnel migrations

apps/production/src/analytics/analytics.controller.ts

@@ -4,9 +4,6 @@ import * as _toNumber from 'lodash/toNumber'
 import * as _pick from 'lodash/pick'
 import * as _includes from 'lodash/includes'
 import * as _keys from 'lodash/keys'
-import * as _size from 'lodash/size'
-import * as _some from 'lodash/some'
-import * as _isString from 'lodash/isString'
 import * as _values from 'lodash/values'
 import * as _map from 'lodash/map'
 import * as _uniqBy from 'lodash/uniqBy'
@@ -64,8 +61,6 @@ import {
   REDIS_PROJECTS_COUNT_KEY,
   REDIS_EVENTS_COUNT_KEY,
   REDIS_SESSION_SALT_KEY,
-  MIN_PAGES_IN_FUNNEL,
-  MAX_PAGES_IN_FUNNEL,
   clickhouse,
 } from '../common/constants'
 import { getGeoDetails, getIPFromHeaders } from '../common/utils'
@@ -279,44 +274,6 @@ const getPIDsArray = (pids, pid) => {
   return pids
 }
 
-const getPagesArray = (rawPages: string): string[] => {
-  try {
-    const pages = JSON.parse(rawPages)
-
-    if (!_isArray(pages)) {
-      throw new UnprocessableEntityException(
-        'An array of pages has to be provided as a pages param',
-      )
-    }
-
-    const size = _size(pages)
-
-    if (size < MIN_PAGES_IN_FUNNEL) {
-      throw new UnprocessableEntityException(
-        `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,
-      )
-    }
-
-    if (size > MAX_PAGES_IN_FUNNEL) {
-      throw new UnprocessableEntityException(
-        `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,
-      )
-    }
-
-    if (_some(pages, page => !_isString(page))) {
-      throw new UnprocessableEntityException(
-        'Pages array must contain string values only',
-      )
-    }
-
-    return pages
-  } catch (e) {
-    throw new UnprocessableEntityException(
-      'Cannot process the provided array of pages',
-    )
-  }
-}
-
 // needed for serving 1x1 px GIF
 const TRANSPARENT_GIF_BUFFER = Buffer.from(
   'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
@@ -477,21 +434,33 @@ export class AnalyticsController {
     @CurrentUserId() uid: string,
     @Headers() headers: { 'x-password'?: string },
   ): Promise<IGetFunnel> {
-    const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data
+    const {
+      pid,
+      period,
+      from,
+      to,
+      timezone = DEFAULT_TIMEZONE,
+      pages,
+      funnelId,
+    } = data
     this.analyticsService.validatePID(pid)
 
     if (!_isEmpty(period)) {
       this.analyticsService.validatePeriod(period)
     }
 
-    const pagesArr = getPagesArray(pages)
-
     await this.analyticsService.checkProjectAccess(
       pid,
       uid,
       headers['x-password'],
     )
 
+    const pagesArr = await this.analyticsService.getPagesArray(
+      pages,
+      funnelId,
+      pid,
+    )
+
     const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
     const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
       from,

apps/production/src/analytics/analytics.service.ts

@@ -47,8 +47,11 @@ import {
   clickhouse,
   REDIS_SESSION_SALT_KEY,
   TRAFFIC_COLUMNS,
+  ALL_COLUMNS,
   CAPTCHA_COLUMNS,
   PERFORMANCE_COLUMNS,
+  MIN_PAGES_IN_FUNNEL,
+  MAX_PAGES_IN_FUNNEL,
 } from '../common/constants'
 import {
   calculateRelativePercentage,
@@ -659,6 +662,60 @@ export class AnalyticsService {
     }
   }
 
+  async getPagesArray(
+    rawPages?: string,
+    funnelId?: string,
+    projectId?: string,
+  ): Promise<string[]> {
+    if (funnelId && projectId) {
+      const funnel = await this.projectService.getFunnel(funnelId, projectId)
+
+      if (!funnel || _isEmpty(funnel)) {
+        throw new UnprocessableEntityException(
+          'The provided funnelId is incorrect',
+        )
+      }
+
+      return funnel.steps
+    }
+
+    try {
+      const pages = JSON.parse(rawPages)
+
+      if (!_isArray(pages)) {
+        throw new UnprocessableEntityException(
+          'An array of pages has to be provided as a pages param',
+        )
+      }
+
+      const size = _size(pages)
+
+      if (size < MIN_PAGES_IN_FUNNEL) {
+        throw new UnprocessableEntityException(
+          `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,
+        )
+      }
+
+      if (size > MAX_PAGES_IN_FUNNEL) {
+        throw new UnprocessableEntityException(
+          `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,
+        )
+      }
+
+      if (_some(pages, page => !_isString(page))) {
+        throw new UnprocessableEntityException(
+          'Pages array must contain string values only',
+        )
+      }
+
+      return pages
+    } catch (e) {
+      throw new UnprocessableEntityException(
+        'Cannot process the provided array of pages',
+      )
+    }
+  }
+
   async getTimeBucketForAllTime(
     pid: string,
     period: string,
@@ -866,15 +923,15 @@ export class AnalyticsService {
       let dropoff = 0
       let eventsPerc = 100
       let eventsPercStep = 100
-      let dropoffPerc = 0
+      let dropoffPercStep = 0
 
       if (index > 0) {
         const prev = data[index - 1]
         events = row.c
         dropoff = prev.c - row.c
         eventsPerc = _round((row.c / data[0].c) * 100, 2)
         eventsPercStep = _round((row.c / prev.c) * 100, 2)
-        dropoffPerc = _round((dropoff / prev.c) * 100, 2)
+        dropoffPercStep = _round((dropoff / prev.c) * 100, 2)
       }
 
       return {
@@ -883,7 +940,7 @@ export class AnalyticsService {
         eventsPerc,
         eventsPercStep,
         dropoff,
-        dropoffPerc,
+        dropoffPercStep,
       }
     })
 
@@ -1133,13 +1190,18 @@ export class AnalyticsService {
   }
 
   async getFilters(pid: string, type: string): Promise<Array<string>> {
-    if (!_includes(TRAFFIC_COLUMNS, type)) {
+    if (!_includes(ALL_COLUMNS, type)) {
       throw new UnprocessableEntityException(
         `The provided type (${type}) is incorrect`,
       )
     }
 
-    const query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+    let query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+
+    if (type === 'ev') {
+      query = `SELECT ${type} FROM customEV WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+    }
+
     const results = await clickhouse
       .query(query, { params: { pid } })
       .toPromise()

apps/production/src/analytics/dto/getFunnels.dto.ts

@@ -26,5 +26,10 @@ export class GetFunnelsDTO {
   @ApiProperty({
     description: 'A stringified array of pages to generate funnel for',
   })
-  pages: string
+  pages?: string
+
+  @ApiProperty({
+    description: 'Funnel ID',
+  })
+  funnelId?: string
 }

apps/production/src/analytics/interfaces/index.ts

@@ -111,7 +111,7 @@ export interface IFunnel {
   eventsPerc: number
   eventsPercStep: number
   dropoff: number
-  dropoffPerc: number
+  dropoffPercStep: number
 }
 
 export interface IGetFunnel {

apps/production/src/common/constants.ts

@@ -134,6 +134,8 @@ const TRAFFIC_COLUMNS = [
   'ca',
 ]
 
+const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']
+
 const CAPTCHA_COLUMNS = ['cc', 'br', 'os', 'dv']
 const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']
 
@@ -174,6 +176,8 @@ const sentryIgnoreErrors: (string | RegExp)[] = [
 const NUMBER_JWT_REFRESH_TOKEN_LIFETIME = Number(JWT_REFRESH_TOKEN_LIFETIME)
 const NUMBER_JWT_ACCESS_TOKEN_LIFETIME = Number(JWT_ACCESS_TOKEN_LIFETIME)
 
+const MAX_FUNNELS = 100
+
 export {
   clickhouse,
   redis,
@@ -222,4 +226,6 @@ export {
   BLOG_POSTS_PATH,
   MIN_PAGES_IN_FUNNEL,
   MAX_PAGES_IN_FUNNEL,
+  MAX_FUNNELS,
+  ALL_COLUMNS,
 }

apps/production/src/project/dto/funnel-create.dto.ts

@@ -0,0 +1,38 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty, Length, ArrayMinSize, ArrayMaxSize } from 'class-validator'
+import {
+  MAX_PAGES_IN_FUNNEL,
+  MIN_PAGES_IN_FUNNEL,
+} from '../../common/constants'
+
+export class FunnelCreateDTO {
+  @ApiProperty({
+    example: 'User sign up funnel',
+    required: true,
+    description: 'A display name for your funnel',
+  })
+  @IsNotEmpty()
+  @Length(1, 50)
+  name: string
+
+  @ApiProperty({
+    example: 'aUn1quEid-3',
+    required: true,
+    description: 'Funnel project ID',
+  })
+  @IsNotEmpty()
+  pid: string
+
+  @ApiProperty({
+    example: ['/', '/signup', '/dashboard'],
+    required: true,
+    description: 'Steps of the funnel',
+  })
+  @ArrayMinSize(MIN_PAGES_IN_FUNNEL, {
+    message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`,
+  })
+  @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, {
+    message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`,
+  })
+  steps: string[] | null
+}

apps/production/src/project/dto/funnel-update.dto.ts

@@ -0,0 +1,47 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+  IsNotEmpty,
+  Length,
+  ArrayMinSize,
+  ArrayMaxSize,
+  IsUUID,
+} from 'class-validator'
+import {
+  MAX_PAGES_IN_FUNNEL,
+  MIN_PAGES_IN_FUNNEL,
+} from '../../common/constants'
+
+export class FunnelUpdateDTO {
+  @IsUUID('4')
+  id: string
+
+  @ApiProperty({
+    example: 'aUn1quEid-3',
+    required: true,
+    description: 'Funnel project ID',
+  })
+  @IsNotEmpty()
+  pid: string
+
+  @ApiProperty({
+    example: 'User sign up funnel',
+    required: true,
+    description: 'A display name for your funnel',
+  })
+  @IsNotEmpty()
+  @Length(1, 50)
+  name: string
+
+  @ApiProperty({
+    example: ['/', '/signup', '/dashboard'],
+    required: true,
+    description: 'Steps of the funnel',
+  })
+  @ArrayMinSize(MIN_PAGES_IN_FUNNEL, {
+    message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`,
+  })
+  @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, {
+    message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`,
+  })
+  steps: string[] | null
+}

apps/production/src/project/dto/index.ts

@@ -12,3 +12,5 @@ export * from './create-project.dto'
 export * from './project.dto'
 export * from './share.dto'
 export * from './share-update.dto'
+export * from './funnel-create.dto'
+export * from './funnel-update.dto'

apps/production/src/project/entity/funnel.entity.ts

@@ -0,0 +1,30 @@
+import {
+  Column,
+  CreateDateColumn,
+  Entity,
+  JoinColumn,
+  ManyToOne,
+  PrimaryGeneratedColumn,
+} from 'typeorm'
+import { Project } from './project.entity'
+
+@Entity()
+export class Funnel {
+  @PrimaryGeneratedColumn('uuid')
+  id: string
+
+  @Column('varchar', { length: 50 })
+  name: string
+
+  @Column({
+    type: 'simple-array',
+  })
+  steps: string[]
+
+  @CreateDateColumn()
+  created: Date
+
+  @ManyToOne(() => Project, project => project.funnels)
+  @JoinColumn()
+  project: Project
+}

apps/production/src/project/entity/index.ts

@@ -1 +1,4 @@
 export * from './project-subscriber.entity'
+export * from './funnel.entity'
+export * from './project.entity'
+export * from './project-share.entity'