diff --git a/.env.example b/.env.example index 1e02c3ce..6c761faa 100644 --- a/.env.example +++ b/.env.example @@ -44,6 +44,7 @@ SMTP_MOCK=true # Selfhosted EMAIL=test@test.com PASSWORD=12345678 +API_KEY= # Swetrix CDN for storing files CDN_URL=http://localhost:5006 diff --git a/Dockerfile b/Dockerfile index 427bc458..dbaa9d5b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,8 @@ ENV TZ=UTC \ CLICKHOUSE_DATABASE=analytics \ API_ORIGINS=\ EMAIL=test@test.com \ - PASSWORD=12345678 + PASSWORD=12345678 \ + API_KEY= RUN apk add --no-cache tzdata && cp /usr/share/zoneinfo/$TZ /etc/localtime WORKDIR /app COPY --from=build /build/package*.json ./ diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts index 66b79a16..6671c37c 100644 --- a/apps/production/src/analytics/analytics.service.ts +++ b/apps/production/src/analytics/analytics.service.ts @@ -824,7 +824,7 @@ export class AnalyticsService { let diff = null if (from && to) { - diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days') + diff = dayjs(to).diff(dayjs(from?.[0]?.created || to), 'days') const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt) diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts index 0f22654c..cab9588e 100644 --- a/apps/production/src/project/project.service.ts +++ b/apps/production/src/project/project.service.ts @@ -838,9 +838,11 @@ export class ProjectService { }) } - async getPIDsWhereAnalyticsDataExists(projectIds: string[]) { + async getPIDsWhereAnalyticsDataExists( + projectIds: string[], + ): Promise { if (_isEmpty(projectIds)) { - return {} + return [] } const params = _reduce( @@ -895,9 +897,9 @@ export class ProjectService { return _map(result, ({ pid }) => pid) } - async getPIDsWhereErrorsDataExists(projectIds: string[]) { + async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise { if (_isEmpty(projectIds)) { - return {} + return [] } const params = _reduce( diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts index 1b1725c8..3af2f00a 100644 --- a/apps/selfhosted/src/analytics/analytics.controller.ts +++ b/apps/selfhosted/src/analytics/analytics.controller.ts @@ -6,9 +6,6 @@ import * as _uniqBy from 'lodash/uniqBy' import * as _round from 'lodash/round' import * as _includes from 'lodash/includes' import * as _keys from 'lodash/keys' -import * as _size from 'lodash/size' -import * as _some from 'lodash/some' -import * as _isString from 'lodash/isString' import * as _values from 'lodash/values' import * as dayjs from 'dayjs' import * as utc from 'dayjs/plugin/utc' @@ -62,8 +59,6 @@ import { REDIS_LOG_CUSTOM_CACHE_KEY, HEARTBEAT_SID_LIFE_TIME, REDIS_SESSION_SALT_KEY, - MIN_PAGES_IN_FUNNEL, - MAX_PAGES_IN_FUNNEL, clickhouse, REDIS_LOG_ERROR_CACHE_KEY, } from '../common/constants' @@ -348,44 +343,6 @@ const getEIDsArray = (eids, eid) => { return [eid] } -const getPagesArray = (rawPages: string): string[] => { - try { - const pages = JSON.parse(rawPages) - - if (!_isArray(pages)) { - throw new UnprocessableEntityException( - 'An array of pages has to be provided as a pages param', - ) - } - - const size = _size(pages) - - if (size < MIN_PAGES_IN_FUNNEL) { - throw new UnprocessableEntityException( - `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`, - ) - } - - if (size > MAX_PAGES_IN_FUNNEL) { - throw new UnprocessableEntityException( - `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`, - ) - } - - if (_some(pages, page => !_isString(page))) { - throw new UnprocessableEntityException( - 'Pages array must contain string values only', - ) - } - - return pages - } catch (e) { - throw new UnprocessableEntityException( - 'Cannot process the provided array of pages', - ) - } -} - // needed for serving 1x1 px GIF const TRANSPARENT_GIF_BUFFER = Buffer.from( 'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=', @@ -406,6 +363,7 @@ export class AnalyticsController { async getData( @Query() data: AnalyticsGET_DTO, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -423,7 +381,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) let newTimebucket = timeBucket let allowedTumebucketForPeriodAll @@ -509,17 +471,34 @@ export class AnalyticsController { async getFunnel( @Query() data: GetFunnelsDTO, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { - const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data + const { + pid, + period, + from, + to, + timezone = DEFAULT_TIMEZONE, + pages, + funnelId, + } = data this.analyticsService.validatePID(pid) if (!_isEmpty(period)) { this.analyticsService.validatePeriod(period) } - const pagesArr = getPagesArray(pages) + const pagesArr = await this.analyticsService.getPagesArray( + pages, + funnelId, + pid, + ) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo( @@ -565,6 +544,7 @@ export class AnalyticsController { async getCustomEventMetadata( @Query() data: GetCustomEventMetadata, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, period } = data this.analyticsService.validatePID(pid) @@ -573,7 +553,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) return this.analyticsService.getCustomEventMetadata(data) } @@ -583,11 +567,16 @@ export class AnalyticsController { async getFilters( @Query() data: GetFiltersDto, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, type } = data this.analyticsService.validatePID(pid) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) return this.analyticsService.getFilters(pid, type) } @@ -597,6 +586,7 @@ export class AnalyticsController { async getChartData( @Query() data: AnalyticsGET_DTO, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -626,7 +616,11 @@ export class AnalyticsController { period, safeTimezone, ) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const paramsData = { params: { @@ -659,6 +653,7 @@ export class AnalyticsController { async getPerfData( @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure }, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -678,7 +673,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) let newTimeBucket = timeBucket let allowedTumebucketForPeriodAll @@ -747,6 +746,7 @@ export class AnalyticsController { async getPerfChartData( @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure }, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -778,7 +778,11 @@ export class AnalyticsController { period, safeTimezone, ) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const paramsData = { params: { @@ -810,6 +814,7 @@ export class AnalyticsController { async getUserFlow( @Query() data: GetUserFlowDTO, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data this.analyticsService.validatePID(pid) @@ -818,7 +823,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) let diff @@ -866,6 +875,7 @@ export class AnalyticsController { async getOverallStats( @Query() data, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pids, @@ -880,7 +890,11 @@ export class AnalyticsController { const validationPromises = _map(pidsArray, async currentPID => { this.analyticsService.validatePID(currentPID) - await this.analyticsService.checkProjectAccess(currentPID, uid) + await this.analyticsService.checkProjectAccess( + currentPID, + uid, + headers['x-password'], + ) }) await Promise.all(validationPromises) @@ -900,6 +914,7 @@ export class AnalyticsController { async getPerformanceOverallStats( @Query() data, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pids, @@ -921,7 +936,11 @@ export class AnalyticsController { const validationPromises = _map(pidsArray, async currentPID => { this.analyticsService.validatePID(currentPID) - await this.analyticsService.checkProjectAccess(currentPID, uid) + await this.analyticsService.checkProjectAccess( + currentPID, + uid, + headers['x-password'], + ) }) await Promise.all(validationPromises) @@ -941,13 +960,18 @@ export class AnalyticsController { async getHeartBeatStats( @Query() data, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pids, pid } = data const pidsArray = getPIDsArray(pids, pid) const validationPromises = _map(pidsArray, async currentPID => { this.analyticsService.validatePID(currentPID) - await this.analyticsService.checkProjectAccess(currentPID, uid) + await this.analyticsService.checkProjectAccess( + currentPID, + uid, + headers['x-password'], + ) }) await Promise.all(validationPromises) @@ -974,11 +998,16 @@ export class AnalyticsController { async getLiveVisitors( @Query() data, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid } = data this.analyticsService.validatePID(pid) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const keys = await redis.keys(`sd:*:${pid}`) @@ -1297,6 +1326,7 @@ export class AnalyticsController { async getSessions( @Query() data: GetSessionsDto, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data this.analyticsService.validatePID(pid) @@ -1305,7 +1335,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const take = this.analyticsService.getSafeNumber(data.take, 30) const skip = this.analyticsService.getSafeNumber(data.skip, 0) @@ -1378,6 +1412,7 @@ export class AnalyticsController { async getSession( @Query() data: GetSessionDto, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -1387,7 +1422,11 @@ export class AnalyticsController { } = data this.analyticsService.validatePID(pid) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const safeTimezone = this.analyticsService.getSafeTimezone(timezone) const result = await this.analyticsService.getSessionDetails( @@ -1404,6 +1443,7 @@ export class AnalyticsController { async getCustomEvents( @Query() data: GetCustomEventsDto, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -1421,7 +1461,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) let newTimeBucket = timeBucket let diff @@ -1505,11 +1549,16 @@ export class AnalyticsController { async getErrorsFilters( @Query() data: GetFiltersDto, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, type } = data this.analyticsService.validatePID(pid) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) return this.analyticsService.getErrorsFilters(pid, type) } @@ -1587,7 +1636,11 @@ export class AnalyticsController { await checkRateLimit(ip, 'error-status', 100, 1800) this.analyticsService.validatePID(pid) - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const eids = getEIDsArray(unprocessedEids, eid) @@ -1601,6 +1654,7 @@ export class AnalyticsController { async getErrors( @Query() data: GetErrorsDto, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -1617,7 +1671,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) const take = this.analyticsService.getSafeNumber(data.take, 30) const skip = this.analyticsService.getSafeNumber(data.skip, 0) @@ -1690,6 +1748,7 @@ export class AnalyticsController { async getError( @Query() data: GetErrorDTO, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { const { pid, @@ -1706,7 +1765,11 @@ export class AnalyticsController { this.analyticsService.validatePeriod(period) } - await this.analyticsService.checkProjectAccess(pid, uid) + await this.analyticsService.checkProjectAccess( + pid, + uid, + headers['x-password'], + ) let timeBucket let diff diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts index 4d986e33..2eb1cff0 100644 --- a/apps/selfhosted/src/analytics/analytics.service.ts +++ b/apps/selfhosted/src/analytics/analytics.service.ts @@ -46,8 +46,14 @@ import { TRAFFIC_COLUMNS, PERFORMANCE_COLUMNS, ERROR_COLUMNS, + MIN_PAGES_IN_FUNNEL, + MAX_PAGES_IN_FUNNEL, } from '../common/constants' -import { millisecondsToSeconds, sumArrays } from '../common/utils' +import { + getFunnelsClickhouse, + millisecondsToSeconds, + sumArrays, +} from '../common/utils' import { PageviewsDTO } from './dto/pageviews.dto' import { EventsDTO } from './dto/events.dto' import { ProjectService } from '../project/project.service' @@ -332,9 +338,13 @@ const isValidOrigin = (origins: string[], origin: string) => { export class AnalyticsService { constructor(private readonly projectService: ProjectService) {} - async checkProjectAccess(pid: string, uid: string | null): Promise { + async checkProjectAccess( + pid: string, + uid: string | null, + password: string | null, + ): Promise { const project = await this.projectService.getRedisProject(pid) - this.projectService.allowedToView(project, uid) + this.projectService.allowedToView(project, uid, password) } checkOrigin(project: Project, origin: string): void { @@ -705,6 +715,62 @@ export class AnalyticsService { } } + async getPagesArray( + rawPages?: string, + funnelId?: string, + projectId?: string, + ): Promise { + if (funnelId && projectId) { + const funnel = this.projectService.formatFunnelFromClickhouse( + await getFunnelsClickhouse(projectId, funnelId), + ) + + if (!funnel || _isEmpty(funnel)) { + throw new UnprocessableEntityException( + 'The provided funnelId is incorrect', + ) + } + + return funnel.steps + } + + try { + const pages = JSON.parse(rawPages) + + if (!_isArray(pages)) { + throw new UnprocessableEntityException( + 'An array of pages has to be provided as a pages param', + ) + } + + const size = _size(pages) + + if (size < MIN_PAGES_IN_FUNNEL) { + throw new UnprocessableEntityException( + `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`, + ) + } + + if (size > MAX_PAGES_IN_FUNNEL) { + throw new UnprocessableEntityException( + `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`, + ) + } + + if (_some(pages, page => !_isString(page))) { + throw new UnprocessableEntityException( + 'Pages array must contain string values only', + ) + } + + return pages + } catch (e) { + throw new UnprocessableEntityException( + 'Cannot process the provided array of pages', + ) + } + } + async getTimeBucketForAllTime( pid: string, period: string, @@ -731,7 +797,7 @@ export class AnalyticsService { let diff = null if (from && to) { - diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days') + diff = dayjs(to).diff(dayjs(from?.[0]?.created || to), 'days') const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt) @@ -932,7 +998,7 @@ export class AnalyticsService { let dropoff = 0 let eventsPerc = 100 let eventsPercStep = 100 - let dropoffPerc = 0 + let dropoffPercStep = 0 if (index > 0) { const prev = data[index - 1] @@ -940,7 +1006,7 @@ export class AnalyticsService { dropoff = prev.c - row.c eventsPerc = _round((row.c / data[0].c) * 100, 2) eventsPercStep = _round((row.c / prev.c) * 100, 2) - dropoffPerc = _round((dropoff / prev.c) * 100, 2) + dropoffPercStep = _round((dropoff / prev.c) * 100, 2) } return { @@ -949,7 +1015,7 @@ export class AnalyticsService { eventsPerc, eventsPercStep, dropoff, - dropoffPerc, + dropoffPercStep, } }) diff --git a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts index afde7917..197895fa 100644 --- a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts +++ b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts @@ -27,4 +27,9 @@ export class GetFunnelsDTO { description: 'A stringified array of pages to generate funnel for', }) pages: string + + @ApiProperty({ + description: 'Funnel ID', + }) + funnelId?: string } diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts index 88de841f..770ada1a 100644 --- a/apps/selfhosted/src/analytics/interfaces/index.ts +++ b/apps/selfhosted/src/analytics/interfaces/index.ts @@ -111,7 +111,7 @@ export interface IFunnel { eventsPerc: number eventsPercStep: number dropoff: number - dropoffPerc: number + dropoffPercStep: number } export interface IGetFunnel { diff --git a/apps/selfhosted/src/auth/auth.module.ts b/apps/selfhosted/src/auth/auth.module.ts index 05e51b70..fb56a08e 100644 --- a/apps/selfhosted/src/auth/auth.module.ts +++ b/apps/selfhosted/src/auth/auth.module.ts @@ -4,7 +4,7 @@ import { PassportModule } from '@nestjs/passport' import { AuthController } from './auth.controller' import { AuthService } from './auth.service' import { - // ApiKeyStrategy, + ApiKeyStrategy, JwtAccessTokenStrategy, JwtRefreshTokenStrategy, } from './strategies' @@ -16,7 +16,7 @@ import { AuthService, JwtAccessTokenStrategy, JwtRefreshTokenStrategy, - // ApiKeyStrategy, + ApiKeyStrategy, ], exports: [AuthService], }) diff --git a/apps/selfhosted/src/auth/auth.service.ts b/apps/selfhosted/src/auth/auth.service.ts index 5284b63f..ebf5e8f9 100644 --- a/apps/selfhosted/src/auth/auth.service.ts +++ b/apps/selfhosted/src/auth/auth.service.ts @@ -15,6 +15,7 @@ import { JWT_ACCESS_TOKEN_SECRET, JWT_ACCESS_TOKEN_LIFETIME, JWT_REFRESH_TOKEN_LIFETIME, + SELFHOSTED_API_KEY, } from '../common/constants' import { getSelfhostedUser, SelfhostedUser } from '../user/entities/user.entity' @@ -111,4 +112,8 @@ export class AuthService { async logout(userId: string, refreshToken: string) { await deleteRefreshTokenClickhouse(userId, refreshToken) } + + isApiKeyValid(apiKey: string): boolean { + return apiKey === SELFHOSTED_API_KEY + } } diff --git a/apps/selfhosted/src/auth/decorators/auth.decorator.ts b/apps/selfhosted/src/auth/decorators/auth.decorator.ts index 25a30e4b..b08e9093 100644 --- a/apps/selfhosted/src/auth/decorators/auth.decorator.ts +++ b/apps/selfhosted/src/auth/decorators/auth.decorator.ts @@ -1,10 +1,6 @@ import { applyDecorators, SetMetadata, UseGuards } from '@nestjs/common' -import { - JwtAccessTokenGuard, - MultiAuthGuard, - RolesGuard, -} from 'src/auth/guards' -import { UserType } from 'src/user/entities/user.entity' +import { JwtAccessTokenGuard, MultiAuthGuard, RolesGuard } from '../guards' +import { UserType } from '../../user/entities/user.entity' import { ROLES_KEY } from './roles.decorator' export const IS_OPTIONAL_AUTH_KEY = 'isOptionalAuth' diff --git a/apps/selfhosted/src/auth/guards/api-key.guard.ts b/apps/selfhosted/src/auth/guards/api-key.guard.ts new file mode 100644 index 00000000..43e58ce2 --- /dev/null +++ b/apps/selfhosted/src/auth/guards/api-key.guard.ts @@ -0,0 +1,5 @@ +import { Injectable } from '@nestjs/common' +import { AuthGuard } from '@nestjs/passport' + +@Injectable() +export class ApiKeyGuard extends AuthGuard('api-key') {} diff --git a/apps/selfhosted/src/auth/guards/multi-auth.guard.ts b/apps/selfhosted/src/auth/guards/multi-auth.guard.ts index 3426269d..96503daa 100644 --- a/apps/selfhosted/src/auth/guards/multi-auth.guard.ts +++ b/apps/selfhosted/src/auth/guards/multi-auth.guard.ts @@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport' import { IS_OPTIONAL_AUTH_KEY } from '../decorators' @Injectable() -export class MultiAuthGuard extends AuthGuard(['jwt-access-token']) { +export class MultiAuthGuard extends AuthGuard(['jwt-access-token', 'api-key']) { constructor(private readonly reflector: Reflector) { super() } diff --git a/apps/selfhosted/src/auth/guards/roles.guard.ts b/apps/selfhosted/src/auth/guards/roles.guard.ts index 210476d1..1a34aab6 100644 --- a/apps/selfhosted/src/auth/guards/roles.guard.ts +++ b/apps/selfhosted/src/auth/guards/roles.guard.ts @@ -3,8 +3,8 @@ import { Reflector } from '@nestjs/core' import { ExtractJwt } from 'passport-jwt' import { verify } from 'jsonwebtoken' -import { UserType } from 'src/user/entities/user.entity' -import { JWT_ACCESS_TOKEN_SECRET } from 'src/common/constants' +import { UserType } from '../../user/entities/user.entity' +import { JWT_ACCESS_TOKEN_SECRET } from '../../common/constants' import { ROLES_KEY } from '../decorators' @Injectable() diff --git a/apps/selfhosted/src/auth/strategies/api-key.strategy.ts b/apps/selfhosted/src/auth/strategies/api-key.strategy.ts new file mode 100644 index 00000000..9e2a4df3 --- /dev/null +++ b/apps/selfhosted/src/auth/strategies/api-key.strategy.ts @@ -0,0 +1,26 @@ +import { Injectable, UnauthorizedException } from '@nestjs/common' +import { PassportStrategy } from '@nestjs/passport' +import { HeaderAPIKeyStrategy } from 'passport-headerapikey' +import { AuthService } from '../auth.service' +import { getSelfhostedUser } from '../../user/entities/user.entity' + +@Injectable() +export class ApiKeyStrategy extends PassportStrategy( + HeaderAPIKeyStrategy, + 'api-key', +) { + constructor(private readonly authService: AuthService) { + super( + { header: 'X-Api-Key', prefix: '' }, + true, + // eslint-disable-next-line consistent-return + async (apiKey: string, done: any) => { + const isValid = this.authService.isApiKeyValid(apiKey) + if (!isValid) return done(new UnauthorizedException(), false) + + const user = await getSelfhostedUser() + done(null, user) + }, + ) + } +} diff --git a/apps/selfhosted/src/auth/strategies/index.ts b/apps/selfhosted/src/auth/strategies/index.ts index 20e4bbdc..f544f42d 100644 --- a/apps/selfhosted/src/auth/strategies/index.ts +++ b/apps/selfhosted/src/auth/strategies/index.ts @@ -1,2 +1,3 @@ export * from './jwt-access-token.strategy' export * from './jwt-refresh-token.strategy' +export * from './api-key.strategy' diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts index 89220bfc..226134ec 100644 --- a/apps/selfhosted/src/common/constants.ts +++ b/apps/selfhosted/src/common/constants.ts @@ -56,6 +56,8 @@ const isDevelopment = process.env.NODE_ENV === 'development' const SELFHOSTED_EMAIL = process.env.EMAIL const SELFHOSTED_PASSWORD = process.env.PASSWORD +const SELFHOSTED_API_KEY = process.env.API_KEY +const SELFHOSTED_API_AUTH_ENABLED = !!SELFHOSTED_API_KEY const UUIDV5_NAMESPACE = '912c64c1-73fd-42b6-859f-785f839a9f68' const DEFAULT_SELFHOSTED_UUID = 'deadbeef-dead-beef-dead-beefdeadbeef' @@ -129,6 +131,8 @@ export { UUIDV5_NAMESPACE, SELFHOSTED_EMAIL, SELFHOSTED_PASSWORD, + SELFHOSTED_API_KEY, + SELFHOSTED_API_AUTH_ENABLED, SELFHOSTED_UUID, REDIS_USERS_COUNT_KEY, REDIS_PROJECTS_COUNT_KEY, diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.ts index 20cf585d..510346f4 100644 --- a/apps/selfhosted/src/common/utils.ts +++ b/apps/selfhosted/src/common/utils.ts @@ -10,6 +10,7 @@ import * as _keys from 'lodash/keys' import * as _toNumber from 'lodash/toNumber' import * as _split from 'lodash/split' import * as _isEmpty from 'lodash/isEmpty' +import * as _isNil from 'lodash/isNil' import * as _head from 'lodash/head' import * as _round from 'lodash/round' import * as _size from 'lodash/size' @@ -28,20 +29,25 @@ import { } from './constants' import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity' import { Project } from '../project/entity/project.entity' +import { Funnel } from '../project/entity/funnel.entity' dayjs.extend(utc) const RATE_LIMIT_REQUESTS_AMOUNT = 3 const RATE_LIMIT_TIMEOUT = 86400 // 24 hours -const allowedToUpdateKeys = [ +const ALLOWED_KEYS = [ 'name', 'origins', 'ipBlacklist', 'active', 'public', + 'isPasswordProtected', + 'passwordHash', ] +const ALLOWED_FUNNEL_KEYS = ['name', 'steps'] + const getRateLimitHash = (ipOrApiKey: string, salt = '') => `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}` @@ -65,9 +71,86 @@ const checkRateLimit = async ( await redis.set(rlHash, 1 + rlCount, 'EX', reqTimeout) } +const getFunnelsClickhouse = async (projectId: string, funnelId = null) => { + const paramsData = { + params: { + projectId, + funnelId, + }, + } + + if (!funnelId) { + const query = + 'SELECT * FROM funnel WHERE projectId = {projectId:FixedString(12)}' + return clickhouse.query(query, paramsData).toPromise() + } + + const query = ` + SELECT + * + FROM funnel + WHERE + projectId = {projectId:FixedString(12)} + AND id = {funnelId:String}` + + const funnel = await clickhouse.query(query, paramsData).toPromise() + + if (_isEmpty(funnel)) { + throw new NotFoundException( + `Funnel ${funnelId} was not found in the database`, + ) + } + + return _head(funnel) +} + +const updateFunnelClickhouse = async (funnel: any) => { + const filtered = _reduce( + _filter(_keys(funnel), key => ALLOWED_FUNNEL_KEYS.includes(key)), + (obj, key) => { + obj[key] = funnel[key] + return obj + }, + {}, + ) + const columns = _keys(filtered) + const values = _values(filtered) + const query = `ALTER table funnel UPDATE ${_join( + _map(columns, (col, id) => `${col}='${values[id]}'`), + ', ', + )} WHERE id='${funnel.id}'` + + return clickhouse.query(query).toPromise() +} + +const deleteFunnelClickhouse = async (id: string) => { + const paramsData = { + params: { + id, + }, + } + + const query = `ALTER table funnel DELETE WHERE id = {id:String}` + return clickhouse.query(query, paramsData).toPromise() +} + +const createFunnelClickhouse = async (funnel: Partial) => { + const paramsData = { + params: { + ...funnel, + }, + } + + const query = `INSERT INTO funnel (*) VALUES ({id:String},{name:String},{steps:String},{projectId:FixedString(12)},'${dayjs + .utc() + .format('YYYY-MM-DD HH:mm:ss')}')` + + return clickhouse.query(query, paramsData).toPromise() +} + const getProjectsClickhouse = async (id = null) => { if (!id) { - const query = 'SELECT * FROM project;' + const query = 'SELECT * FROM project ORDER BY created ASC;' return clickhouse.query(query).toPromise() } @@ -87,9 +170,9 @@ const getProjectsClickhouse = async (id = null) => { return _head(project) } -const updateProjectClickhouse = async (project: object) => { +const updateProjectClickhouse = async (project: any) => { const filtered = _reduce( - _filter(_keys(project), key => allowedToUpdateKeys.includes(key)), + _filter(_keys(project), key => ALLOWED_KEYS.includes(key)), (obj, key) => { obj[key] = project[key] return obj @@ -98,11 +181,9 @@ const updateProjectClickhouse = async (project: object) => { ) const columns = _keys(filtered) const values = _values(filtered) - // @ts-ignore const query = `ALTER table project UPDATE ${_join( _map(columns, (col, id) => `${col}='${values[id]}'`), ', ', - // @ts-ignore )} WHERE id='${project.id}'` return clickhouse.query(query).toPromise() } @@ -129,9 +210,15 @@ const calculateRelativePercentage = ( return _round((1 - newVal / oldVal) * -100, round) } -const deleteProjectClickhouse = async id => { - const query = `ALTER table project DELETE WHERE id='${id}'` - return clickhouse.query(query).toPromise() +const deleteProjectClickhouse = async (id: string) => { + const paramsData = { + params: { + id, + }, + } + + const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}` + return clickhouse.query(query, paramsData).toPromise() } const createProjectClickhouse = async (project: Partial) => { @@ -261,17 +348,17 @@ const updateUserClickhouse = async (user: IClickhouseUser) => { let query = 'ALTER table sfuser UPDATE ' let separator = '' - if (user.timezone) { + if (!_isNil(user.timezone)) { query += `${separator}timezone={timezone:String}` separator = ', ' } - if (user.timeFormat) { + if (!_isNil(user.timeFormat)) { query += `${separator}timeFormat={timeFormat:String}` separator = ', ' } - if (user.showLiveVisitorsInTitle) { + if (!_isNil(user.showLiveVisitorsInTitle)) { query += `${separator}showLiveVisitorsInTitle={showLiveVisitorsInTitle:Int8}` separator = ', ' } @@ -385,4 +472,8 @@ export { getGeoDetails, getIPFromHeaders, sumArrays, + getFunnelsClickhouse, + updateFunnelClickhouse, + deleteFunnelClickhouse, + createFunnelClickhouse, } diff --git a/apps/selfhosted/src/project/dto/funnel-create.dto.ts b/apps/selfhosted/src/project/dto/funnel-create.dto.ts new file mode 100644 index 00000000..6d2ab444 --- /dev/null +++ b/apps/selfhosted/src/project/dto/funnel-create.dto.ts @@ -0,0 +1,38 @@ +import { ApiProperty } from '@nestjs/swagger' +import { IsNotEmpty, Length, ArrayMinSize, ArrayMaxSize } from 'class-validator' +import { + MAX_PAGES_IN_FUNNEL, + MIN_PAGES_IN_FUNNEL, +} from '../../common/constants' + +export class FunnelCreateDTO { + @ApiProperty({ + example: 'User sign up funnel', + required: true, + description: 'A display name for your funnel', + }) + @IsNotEmpty() + @Length(1, 50) + name: string + + @ApiProperty({ + example: 'aUn1quEid-3', + required: true, + description: 'Funnel project ID', + }) + @IsNotEmpty() + pid: string + + @ApiProperty({ + example: ['/', '/signup', '/dashboard'], + required: true, + description: 'Steps of the funnel', + }) + @ArrayMinSize(MIN_PAGES_IN_FUNNEL, { + message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`, + }) + @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, { + message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`, + }) + steps: string[] | null +} diff --git a/apps/selfhosted/src/project/dto/funnel-update.dto.ts b/apps/selfhosted/src/project/dto/funnel-update.dto.ts new file mode 100644 index 00000000..31a7a3c3 --- /dev/null +++ b/apps/selfhosted/src/project/dto/funnel-update.dto.ts @@ -0,0 +1,47 @@ +import { ApiProperty } from '@nestjs/swagger' +import { + IsNotEmpty, + Length, + ArrayMinSize, + ArrayMaxSize, + IsUUID, +} from 'class-validator' +import { + MAX_PAGES_IN_FUNNEL, + MIN_PAGES_IN_FUNNEL, +} from '../../common/constants' + +export class FunnelUpdateDTO { + @IsUUID('4') + id: string + + @ApiProperty({ + example: 'aUn1quEid-3', + required: true, + description: 'Funnel project ID', + }) + @IsNotEmpty() + pid: string + + @ApiProperty({ + example: 'User sign up funnel', + required: true, + description: 'A display name for your funnel', + }) + @IsNotEmpty() + @Length(1, 50) + name: string + + @ApiProperty({ + example: ['/', '/signup', '/dashboard'], + required: true, + description: 'Steps of the funnel', + }) + @ArrayMinSize(MIN_PAGES_IN_FUNNEL, { + message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`, + }) + @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, { + message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`, + }) + steps: string[] | null +} diff --git a/apps/selfhosted/src/project/dto/index.ts b/apps/selfhosted/src/project/dto/index.ts index 2c80f699..1308d459 100644 --- a/apps/selfhosted/src/project/dto/index.ts +++ b/apps/selfhosted/src/project/dto/index.ts @@ -1,2 +1,6 @@ export * from './create-project.dto' export * from './project.dto' +export * from './update-project.dto' +export * from './project-password.dto' +export * from './funnel-create.dto' +export * from './funnel-update.dto' diff --git a/apps/selfhosted/src/project/dto/project-password.dto.ts b/apps/selfhosted/src/project/dto/project-password.dto.ts new file mode 100644 index 00000000..9c033b58 --- /dev/null +++ b/apps/selfhosted/src/project/dto/project-password.dto.ts @@ -0,0 +1,29 @@ +import { + IsOptional, + IsString, + MaxLength, + MinLength, + ValidateIf, +} from 'class-validator' + +export const MAX_PROJECT_PASSWORD_LENGTH = 80 + +export class ProjectPasswordDto { + @IsString() + @MaxLength(MAX_PROJECT_PASSWORD_LENGTH, { + message: 'Max length is $constraint1 characters', + }) + @MinLength(1, { message: 'Min length is $constraint1 characters' }) + @IsOptional() + password?: string | null + + @IsOptional() + @ValidateIf(o => o.isPasswordProtected !== null) + isPasswordProtected?: boolean | null +} + +export class GetProtectedProjectDto { + @IsOptional() + @ValidateIf(o => o.password !== null) + password?: string | null +} diff --git a/apps/selfhosted/src/project/dto/update-project.dto.ts b/apps/selfhosted/src/project/dto/update-project.dto.ts new file mode 100644 index 00000000..a6dbcde0 --- /dev/null +++ b/apps/selfhosted/src/project/dto/update-project.dto.ts @@ -0,0 +1,23 @@ +import { IntersectionType, PartialType } from '@nestjs/mapped-types' +import { ApiProperty } from '@nestjs/swagger' +import { ProjectDTO } from './project.dto' +import { ProjectPasswordDto } from './project-password.dto' + +export class UpdateProjectDto extends IntersectionType( + PartialType(ProjectDTO), + ProjectPasswordDto, +) { + @ApiProperty({ + required: false, + description: + "The project's state. If enabled - all the incoming analytics data will be saved.", + }) + active?: boolean + + @ApiProperty({ + required: false, + description: + "When true, anyone on the internet (including Google) would be able to see the project's Dashboard.", + }) + public?: boolean +} diff --git a/apps/selfhosted/src/project/entity/funnel.entity.ts b/apps/selfhosted/src/project/entity/funnel.entity.ts new file mode 100644 index 00000000..eb3614c2 --- /dev/null +++ b/apps/selfhosted/src/project/entity/funnel.entity.ts @@ -0,0 +1,11 @@ +export class Funnel { + id: string + + name: string + + steps: string[] + + created: Date + + projectId: string +} diff --git a/apps/selfhosted/src/project/entity/project.entity.ts b/apps/selfhosted/src/project/entity/project.entity.ts index dbe0df79..420ee292 100644 --- a/apps/selfhosted/src/project/entity/project.entity.ts +++ b/apps/selfhosted/src/project/entity/project.entity.ts @@ -11,5 +11,9 @@ export class Project { public: boolean + isPasswordProtected: boolean + + passwordHash?: string + created: Date } diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.ts index 75f86f2f..7c7088f6 100644 --- a/apps/selfhosted/src/project/project.controller.ts +++ b/apps/selfhosted/src/project/project.controller.ts @@ -11,16 +11,24 @@ import { BadRequestException, HttpCode, NotFoundException, + Headers, + UnauthorizedException, + Patch, } from '@nestjs/common' +import { v4 as uuidv4 } from 'uuid' import { ApiTags, ApiQuery, ApiResponse, ApiBearerAuth } from '@nestjs/swagger' import * as _isEmpty from 'lodash/isEmpty' import * as _map from 'lodash/map' import * as _trim from 'lodash/trim' import * as _size from 'lodash/size' import * as _split from 'lodash/split' +import * as _isBoolean from 'lodash/isBoolean' +import * as _omit from 'lodash/omit' +import * as _reduce from 'lodash/reduce' import * as _head from 'lodash/head' import * as _includes from 'lodash/includes' import * as dayjs from 'dayjs' +import { hash } from 'bcrypt' import { JwtAccessTokenGuard } from '../auth/guards' import { Auth } from '../auth/decorators' @@ -36,7 +44,13 @@ import { RolesGuard } from '../auth/guards/roles.guard' import { Pagination } from '../common/pagination/pagination' import { Project } from './entity/project.entity' import { CurrentUserId } from '../auth/decorators/current-user-id.decorator' -import { ProjectDTO, CreateProjectDTO } from './dto' +import { + ProjectDTO, + CreateProjectDTO, + UpdateProjectDto, + FunnelCreateDTO, + FunnelUpdateDTO, +} from './dto' import { AppLoggerService } from '../logger/logger.service' import { isValidPID, clickhouse } from '../common/constants' import { @@ -44,7 +58,12 @@ import { createProjectClickhouse, updateProjectClickhouse, deleteProjectClickhouse, + getFunnelsClickhouse, + deleteFunnelClickhouse, + createFunnelClickhouse, + updateFunnelClickhouse, } from '../common/utils' +import { Funnel } from './entity/funnel.entity' @ApiTags('Project') @Controller('project') @@ -81,8 +100,35 @@ export class ProjectController { _map(formatted, ({ id }) => id), ) + const funnelsData = await Promise.allSettled( + pidsWithData.map(async pid => { + return { + pid, + data: await getFunnelsClickhouse(pid), + } + }), + ) + + const funnelsMap = _reduce( + funnelsData, + (acc, { status, value }) => { + if (status !== 'fulfilled') { + return acc + } + + return { + ...acc, + [value.pid]: this.projectService.formatFunnelsFromClickhouse( + value.data, + ), + } + }, + {}, + ) + const results = _map(formatted, p => ({ - ...p, + ..._omit(p, ['passwordHash']), + funnels: funnelsMap[p.id], isOwner: true, isLocked: false, isDataExists: _includes(pidsWithData, p?.id), @@ -168,6 +214,153 @@ export class ProjectController { } } + @Post('/funnel') + @ApiResponse({ status: 201 }) + @Auth([], true) + async createFunnel( + @Body() funnelDTO: FunnelCreateDTO, + @CurrentUserId() userId: string, + ): Promise { + this.logger.log({ funnelDTO, userId }, 'POST /project/funnel') + + if (!userId) { + throw new UnauthorizedException('Please auth first') + } + + const project = getProjectsClickhouse(funnelDTO.pid) + + if (!project) { + throw new NotFoundException('Project not found.') + } + + const funnel = new Funnel() + funnel.id = uuidv4() + funnel.name = funnelDTO.name + funnel.steps = _map(funnelDTO.steps, _trim) + funnel.projectId = funnelDTO.pid + + const formatted = this.projectService.formatFunnelToClickhouse(funnel) + + await createFunnelClickhouse(formatted) + + return { + ...funnel, + pid: funnelDTO.pid, + project: _omit(this.projectService.formatFromClickhouse(project), [ + 'passwordHash', + ]), + } + } + + @Patch('/funnel') + @ApiResponse({ status: 200 }) + @Auth([], true) + async updateFunnel( + @Body() funnelDTO: FunnelUpdateDTO, + @CurrentUserId() userId: string, + ): Promise { + this.logger.log({ funnelDTO, userId }, 'PATCH /project/funnel') + + if (!userId) { + throw new UnauthorizedException('Please auth first') + } + + const project = getProjectsClickhouse(funnelDTO.pid) + + if (!project) { + throw new NotFoundException('Project not found.') + } + + const oldFunnel = this.projectService.formatFunnelFromClickhouse( + await getFunnelsClickhouse(funnelDTO.pid, funnelDTO.id), + ) + + if (!oldFunnel) { + throw new NotFoundException('Funnel not found.') + } + + await updateFunnelClickhouse( + this.projectService.formatFunnelToClickhouse({ + id: funnelDTO.id, + name: funnelDTO.name, + steps: funnelDTO.steps, + } as Funnel), + ) + } + + @Delete('/funnel/:id/:pid') + @ApiResponse({ status: 200 }) + @Auth([], true) + async deleteFunnel( + @Param('id') id: string, + @Param('pid') pid: string, + @CurrentUserId() userId: string, + ): Promise { + this.logger.log({ id, userId }, 'PATCH /project/funnel') + + if (!userId) { + throw new UnauthorizedException('Please auth first') + } + + const oldFunnel = getFunnelsClickhouse(pid, id) + + if (!oldFunnel) { + throw new NotFoundException('Funnel not found.') + } + + await deleteFunnelClickhouse(id) + } + + @Get('/funnels/:pid') + @ApiResponse({ status: 200 }) + @Auth([], true) + async getFunnels( + @Param('pid') pid: string, + @CurrentUserId() userId: string, + @Headers() headers: { 'x-password'?: string }, + ): Promise { + this.logger.log({ pid, userId }, 'PATCH /project/funnel') + + if (!userId) { + throw new UnauthorizedException('Please auth first') + } + + const project = await getProjectsClickhouse(pid) + + if (!project) { + throw new NotFoundException('Project not found.') + } + + this.projectService.allowedToView(project, userId, headers['x-password']) + + return getFunnelsClickhouse(pid) + } + + @Get('password/:projectId') + @Auth([], true, true) + @ApiResponse({ status: 200, type: Project }) + async checkPassword( + @Param('projectId') projectId: string, + @CurrentUserId() userId: string, + @Headers() headers: { 'x-password'?: string }, + ): Promise { + this.logger.log({ projectId }, 'GET /project/password/:projectId') + + const project = await getProjectsClickhouse(projectId) + + if (!project) { + throw new NotFoundException('Project not found.') + } + + try { + this.projectService.allowedToView(project, userId, headers['x-password']) + } catch { + return false + } + + return true + } + @Delete('/partially/:pid') @ApiQuery({ name: 'from', @@ -259,7 +452,7 @@ export class ProjectController { @ApiResponse({ status: 200, type: Project }) async update( @Param('id') id: string, - @Body() projectDTO: ProjectDTO, + @Body() projectDTO: UpdateProjectDto, @CurrentUserId() uid: string, ): Promise { this.logger.log({ projectDTO, uid, id }, 'PUT /project/:id') @@ -270,11 +463,37 @@ export class ProjectController { throw new NotFoundException() } - project.active = projectDTO.active - project.origins = _map(projectDTO.origins, _trim) - project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim) - project.name = projectDTO.name - project.public = projectDTO.public + if (_isBoolean(projectDTO.public)) { + project.public = projectDTO.public + } + + if (_isBoolean(projectDTO.active)) { + project.active = projectDTO.active + } + + if (projectDTO.origins) { + project.origins = _map(projectDTO.origins, _trim) + } + + if (projectDTO.ipBlacklist) { + project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim) + } + + if (projectDTO.name) { + project.name = _trim(projectDTO.name) + } + + if (_isBoolean(projectDTO.isPasswordProtected)) { + if (projectDTO.isPasswordProtected) { + if (projectDTO.password) { + project.isPasswordProtected = true + project.passwordHash = await hash(projectDTO.password, 10) + } + } else { + project.isPasswordProtected = false + project.passwordHash = null + } + } await updateProjectClickhouse( this.projectService.formatToClickhouse(project), @@ -283,7 +502,7 @@ export class ProjectController { // await updateProjectRedis(id, project) await deleteProjectRedis(id) - return project + return _omit(project, ['passwordHash']) } @Get('/:id') @@ -292,6 +511,7 @@ export class ProjectController { async getOne( @Param('id') id: string, @CurrentUserId() uid: string, + @Headers() headers: { 'x-password'?: string }, ): Promise { this.logger.log({ id }, 'GET /project/:id') if (!isValidPID(id)) { @@ -306,7 +526,14 @@ export class ProjectController { throw new NotFoundException('Project was not found in the database') } - this.projectService.allowedToView(project, uid) + if (project.isPasswordProtected && _isEmpty(headers['x-password'])) { + return { + isPasswordProtected: true, + id: project.id, + } + } + + this.projectService.allowedToView(project, uid, headers['x-password']) const isDataExists = !_isEmpty( await this.projectService.getPIDsWhereAnalyticsDataExists([id]), @@ -316,8 +543,11 @@ export class ProjectController { await this.projectService.getPIDsWhereErrorsDataExists([id]), ) + const funnels = await getFunnelsClickhouse(id) + return this.projectService.formatFromClickhouse({ - ...project, + ..._omit(project, ['passwordHash']), + funnels: this.projectService.formatFunnelsFromClickhouse(funnels), isDataExists, isErrorDataExists, }) diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.ts index e845cffb..ed8c7504 100644 --- a/apps/selfhosted/src/project/project.service.ts +++ b/apps/selfhosted/src/project/project.service.ts @@ -18,6 +18,7 @@ import * as _isNull from 'lodash/isNull' import * as _split from 'lodash/split' import * as _trim from 'lodash/trim' import * as _reduce from 'lodash/reduce' +import { compareSync } from 'bcrypt' import { Project } from './entity/project.entity' import { ProjectDTO } from './dto/project.dto' @@ -31,6 +32,8 @@ import { redisProjectCacheTimeout, } from '../common/constants' import { getProjectsClickhouse } from '../common/utils' +import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto' +import { Funnel } from './entity/funnel.entity' // A list of characters that can be used in a Project ID const LEGAL_PID_CHARACTERS = @@ -79,7 +82,22 @@ export class ProjectService { return project } - allowedToView(project: Project, uid: string | null): void { + allowedToView( + project: Project, + uid: string | null, + password?: string | null, + ): void { + if (project.isPasswordProtected && password) { + if ( + _size(password) <= MAX_PROJECT_PASSWORD_LENGTH && + compareSync(password, project.passwordHash) + ) { + return null + } + + throw new ConflictException('Incorrect password') + } + if (project.public || uid) { return null } @@ -99,9 +117,11 @@ export class ProjectService { } } - async getPIDsWhereAnalyticsDataExists(projectIds: string[]) { + async getPIDsWhereAnalyticsDataExists( + projectIds: string[], + ): Promise { if (_isEmpty(projectIds)) { - return {} + return [] } const params = _reduce( @@ -156,9 +176,9 @@ export class ProjectService { return _map(result, ({ pid }) => pid) } - async getPIDsWhereErrorsDataExists(projectIds: string[]) { + async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise { if (_isEmpty(projectIds)) { - return {} + return [] } const params = _reduce( @@ -230,6 +250,7 @@ export class ProjectService { const updProject = { ...project } updProject.active = Number(updProject.active) updProject.public = Number(updProject.public) + updProject.isPasswordProtected = Number(updProject.isPasswordProtected) if (!_isNull(updProject.origins)) { updProject.origins = _isString(updProject.origins) @@ -250,6 +271,7 @@ export class ProjectService { const updProject = { ...project } updProject.active = Boolean(updProject.active) updProject.public = Boolean(updProject.public) + updProject.isPasswordProtected = Boolean(updProject.isPasswordProtected) updProject.origins = _isNull(updProject.origins) ? [] @@ -262,7 +284,38 @@ export class ProjectService { return updProject } - validateProject(projectDTO: ProjectDTO, creatingProject = false) { + formatFunnelToClickhouse(funnel: Funnel): any { + const updFunnel = { ...funnel } + updFunnel.name = _trim(funnel.name) + updFunnel.steps = _isString(updFunnel.steps) + ? updFunnel.steps + : _join(updFunnel.steps, ',') + + return updFunnel + } + + formatFunnelsFromClickhouse(funnels: any[]): Funnel[] { + if (_isEmpty(funnels)) { + return [] + } + + return _map(funnels, this.formatFunnelFromClickhouse) + } + + formatFunnelFromClickhouse(funnel: any): Funnel { + const updFunnel = { ...funnel } + + updFunnel.steps = _isNull(updFunnel.steps) + ? [] + : _split(updFunnel.steps, ',') + + return updFunnel + } + + validateProject( + projectDTO: ProjectDTO | UpdateProjectDto, + creatingProject = false, + ) { if (_size(projectDTO.name) > 50) throw new UnprocessableEntityException('The project name is too long') diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 02ae8f9e..beca87ac 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,11 +1,11 @@ version: '3.8' services: redis: - image: redis:7.0.11-alpine + image: redis:7.2-alpine ports: - 6379:6379 clickhouse: - image: clickhouse/clickhouse-server:23.4.2.11-alpine + image: clickhouse/clickhouse-server:23.8-alpine environment: CLICKHOUSE_DB: 'analytics' ports: diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml index 438a2daf..3af77d32 100644 --- a/docker-compose-prod.yml +++ b/docker-compose-prod.yml @@ -58,7 +58,7 @@ services: - 'mariadb' - 'clickhouse' redis: - image: redis:7.0.11-alpine + image: redis:7.2-alpine restart: always environment: - REDIS_PORT=6379 @@ -77,7 +77,7 @@ services: volumes: - swetrix-users-data:/var/lib/mysql clickhouse: - image: clickhouse/clickhouse-server:23.4.2.11-alpine + image: clickhouse/clickhouse-server:23.8-alpine container_name: clickhouse environment: - CLICKHOUSE_DATABASE=analytics diff --git a/docker-compose.yml b/docker-compose.yml index 4fae851f..adbcaffb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,7 +36,7 @@ services: - 'redis' - 'clickhouse' redis: - image: redis:7.0.11-alpine + image: redis:7.2-alpine restart: always environment: - REDIS_PORT=6379 @@ -47,7 +47,7 @@ services: # volumes: # - '/opt/redis-volume:/data' clickhouse: - image: clickhouse/clickhouse-server:23.4.2.11-alpine + image: clickhouse/clickhouse-server:23.8-alpine container_name: clickhouse environment: - CLICKHOUSE_DATABASE=analytics diff --git a/migrations/clickhouse/2023_05_19.js b/migrations/clickhouse/2023_05_19.js index ca8201f2..1887deed 100644 --- a/migrations/clickhouse/2023_05_19.js +++ b/migrations/clickhouse/2023_05_19.js @@ -100,7 +100,7 @@ const queries = [ ) ENGINE = MergeTree() PARTITION BY toYYYYMM(created) - ORDER BY (pid, created);` + ORDER BY (pid, created);`, `INSERT INTO ${dbName}.captcha_temp (*) SELECT * FROM ${dbName}.captcha`, diff --git a/migrations/clickhouse/initialise_selfhosted.js b/migrations/clickhouse/initialise_selfhosted.js index 17238a46..6517518d 100644 --- a/migrations/clickhouse/initialise_selfhosted.js +++ b/migrations/clickhouse/initialise_selfhosted.js @@ -11,6 +11,8 @@ const CLICKHOUSE_INIT_QUERIES = [ ipBlacklist Nullable(String), active Int8, public Int8, + isPasswordProtected Int8, + passwordHash Nullable(String), created DateTime ) ENGINE = MergeTree() @@ -34,6 +36,17 @@ const CLICKHOUSE_INIT_QUERIES = [ ) ENGINE = MergeTree() PRIMARY KEY id;`, + + `CREATE TABLE IF NOT EXISTS ${dbName}.funnel + ( + id String, + name String, + steps String, + projectId FixedString(12), + created DateTime + ) + ENGINE = MergeTree() + PRIMARY KEY id;`, ] const initialiseSelfhosted = async () => { @@ -41,7 +54,9 @@ const initialiseSelfhosted = async () => { await initialiseDatabase() await queriesRunner(CLICKHOUSE_INIT_QUERIES) } catch (reason) { - console.error(`[ERROR] Error occured whilst initialising the database: ${reason}`) + console.error( + `[ERROR] Error occured whilst initialising the database: ${reason}`, + ) } } diff --git a/migrations/clickhouse/selfhosted_2023_05_28.js b/migrations/clickhouse/selfhosted_2023_05_28.js index 6c034d0f..453632c8 100644 --- a/migrations/clickhouse/selfhosted_2023_05_28.js +++ b/migrations/clickhouse/selfhosted_2023_05_28.js @@ -7,7 +7,7 @@ const queries = [ ( id String, timezone Nullable(String), - timeFormat Nullable(String) + timeFormat Nullable(String), showLiveVisitorsInTitle Int8 ) ENGINE = MergeTree() diff --git a/migrations/clickhouse/selfhosted_2023_10_11.js b/migrations/clickhouse/selfhosted_2023_10_11.js index 1275bd50..dbf1821d 100644 --- a/migrations/clickhouse/selfhosted_2023_10_11.js +++ b/migrations/clickhouse/selfhosted_2023_10_11.js @@ -1,7 +1,7 @@ const { queriesRunner, dbName } = require('./setup') const queries = [ - // Added 'showLiveVisitorsInTitle' column + // Made 'showLiveVisitorsInTitle' column nullable `DROP TABLE IF EXISTS ${dbName}.sfuser_temp`, `CREATE TABLE IF NOT EXISTS ${dbName}.sfuser_temp ( diff --git a/migrations/clickhouse/selfhosted_2024_05_13.js b/migrations/clickhouse/selfhosted_2024_05_13.js new file mode 100644 index 00000000..d07192b3 --- /dev/null +++ b/migrations/clickhouse/selfhosted_2024_05_13.js @@ -0,0 +1,41 @@ +const { queriesRunner, dbName } = require('./setup') + +const queries = [ + // Added password-related columns + `DROP TABLE IF EXISTS ${dbName}.project_temp`, + `CREATE TABLE IF NOT EXISTS ${dbName}.project_temp + ( + id FixedString(12), + name String, + origins Nullable(String), + ipBlacklist Nullable(String), + active Int8, + public Int8, + isPasswordProtected Int8, + passwordHash Nullable(String), + created DateTime + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(created) + ORDER BY (created);`, + + `INSERT INTO ${dbName}.project_temp (*) + SELECT id, name, origins, ipBlacklist, active, public, 0, NULL, created FROM ${dbName}.project`, + + `DROP TABLE ${dbName}.project`, + `RENAME TABLE ${dbName}.project_temp TO ${dbName}.project`, + + // Added funnel table + `CREATE TABLE IF NOT EXISTS ${dbName}.funnel + ( + id String, + name String, + steps String, + projectId FixedString(12), + created DateTime + ) + ENGINE = MergeTree() + PRIMARY KEY id;`, +] + +queriesRunner(queries) diff --git a/migrations/clickhouse/selfhosted_v2_to_v3.js b/migrations/clickhouse/selfhosted_v2_to_v3.js new file mode 100644 index 00000000..595fb947 --- /dev/null +++ b/migrations/clickhouse/selfhosted_v2_to_v3.js @@ -0,0 +1,48 @@ +const childProcess = require('child_process') +const path = require('path') + +function runScript(scriptPath) { + return new Promise((resolve, reject) => { + const fullPath = path.join(__dirname, scriptPath) + console.log('[INFO] Running script: ' + fullPath) + + const process = childProcess.fork(fullPath) + let invoked = false + + // listen for errors as they may prevent the exit event from firing + process.on('error', function (err) { + if (invoked) return + invoked = true + reject(err) + }) + + // execute the callback once the process has finished running + process.on('exit', function (code) { + if (invoked) return + invoked = true + const err = code === 0 ? null : new Error('exit code ' + code) + err ? reject(err) : resolve() + }) + }) +} + +async function runMigrations() { + try { + await runScript('./initialise_database.js') + await runScript('./initialise_selfhosted.js') + await runScript('./2023_05_19.js') + await runScript('./2023_06_26.js') + await runScript('./2023_10_13.js') + await runScript('./2023_10_18.js') + await runScript('./2024_03_30.js') + await runScript('./selfhosted_2023_05_28.js') + await runScript('./selfhosted_2023_10_11.js') + await runScript('./selfhosted_2024_05_13.js') + + console.log('[INFO] SELFHOSTED MIGRATIONS FINISHED') + } catch (err) { + console.error('Error running migration:', err) + } +} + +runMigrations()