From 94967d45e13b55570c039b4fdeaf00f2e35accf0 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 20 Jul 2023 21:13:29 -0700 Subject: [PATCH 1/7] update funnels relational query --- lib/prisma.ts | 41 ++++++++++++------- .../analytics/pageview/getPageviewFunnel.ts | 36 ++++++++-------- 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/lib/prisma.ts b/lib/prisma.ts index 875f5897..2197a1ae 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -44,6 +44,18 @@ function getAddMinutesQuery(field: string, minutes: number) { } } +function getDropoffQuery() { + const db = getDatabaseType(process.env.DATABASE_URL); + + if (db === POSTGRESQL) { + return `round((1.0 - count::numeric/lag(count, 1) over ()),2)`; + } + + if (db === MYSQL) { + return `round((1.0 - count/LAG(count, 1) OVER (ORDER BY level)),2)`; + } +} + function getDateQuery(field: string, unit: string, timezone?: string): string { const db = getDatabaseType(process.env.DATABASE_URL); @@ -147,29 +159,27 @@ function getFunnelQuery( return urls.reduce( (pv, cv, i) => { const levelNumber = i + 1; - const start = i > 0 ? ',' : ''; + const startSum = i > 0 ? 'union ' : ''; + const startFilter = i > 0 ? ', ' : ''; if (levelNumber >= 2) { pv.levelQuery += `\n , level${levelNumber} AS ( - select cl.*, - l0.created_at level_${levelNumber}_created_at, - l0.url_path as level_${levelNumber}_url - from level${i} cl - left join website_event l0 - on cl.session_id = l0.session_id - and l0.created_at between cl.level_${i}_created_at - and ${getAddMinutesQuery(`cl.level_${i}_created_at`, windowMinutes)} - and l0.referrer_path = $${i + initParamLength} - and l0.url_path = $${levelNumber + initParamLength} - and created_at between $2 and $3 - and website_id = $1${toUuid()} + select distinct l.session_id, we.created_at + from level${i} l + join website_event we + on l.session_id = we.session_id + where we.created_at between l.created_at + and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} + and we.referrer_path = $${i + initParamLength} + and we.url_path = $${levelNumber + initParamLength} + and we.website_id = $1${toUuid()} )`; } - pv.sumQuery += `\n${start}SUM(CASE WHEN level_${levelNumber}_url is not null THEN 1 ELSE 0 END) AS level${levelNumber}`; + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - pv.urlFilterQuery += `\n${start}$${levelNumber + initParamLength} `; + pv.urlFilterQuery += `${startFilter}$${levelNumber + initParamLength} `; return pv; }, @@ -212,6 +222,7 @@ async function rawQuery(query: string, params: never[] = []): Promise { export default { ...prisma, getAddMinutesQuery, + getDropoffQuery, getDateQuery, getTimestampInterval, getFilterQuery, diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index bfd921c1..ef52101f 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -34,7 +34,7 @@ async function relationalQuery( }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getFunnelQuery, toUuid } = prisma; + const { rawQuery, getFunnelQuery, toUuid, getDropoffQuery } = prisma; const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes); const params: any = [websiteId, startDate, endDate, ...urls]; @@ -47,14 +47,18 @@ async function relationalQuery( and website_id = $1${toUuid()} and created_at between $2 and $3 ),level1 AS ( - select distinct session_id, url_path as level_1_url, created_at as level_1_created_at + select distinct session_id, created_at from level0 where url_path = $4 - )${levelQuery} - - SELECT ${sumQuery} - from level${urls.length}; - `, + )${levelQuery}, levelCount as ( + ${sumQuery} + order by level) + select + level, + count, + ${getDropoffQuery()} as drop_off + from levelCount; + `, params, ).then((a: { [key: string]: number }) => { return urls.map((b, i) => ({ x: b, y: a[0][`level${i + 1}`] || 0 })); @@ -77,7 +81,7 @@ async function clickhouseQuery( > { const { windowMinutes, startDate, endDate, urls } = criteria; const { rawQuery, getBetweenDates, getFunnelQuery } = clickhouse; - const { columnsQuery, conditionQuery, urlParams } = getFunnelQuery(urls); + const { columnsQuery, urlParams } = getFunnelQuery(urls); const params = { websiteId, @@ -87,22 +91,22 @@ async function clickhouseQuery( return rawQuery<{ level: number; count: number }[]>( ` - SELECT level, + select level, count(*) AS count - FROM ( - SELECT session_id, + from ( + select session_id, windowFunnel({window:UInt32}, 'strict_increase') ( created_at ${columnsQuery} ) AS level - FROM website_event - WHERE website_id = {websiteId:UUID} + from website_event + where website_id = {websiteId:UUID} and ${getBetweenDates('created_at', startDate, endDate)} - GROUP BY 1 + group by 1 ) - GROUP BY level - ORDER BY level ASC; + group by level + order by level asc; `, params, ).then(results => { From 858f4655661935e31adfa2b8238be338e382d9be Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Fri, 21 Jul 2023 00:14:37 -0700 Subject: [PATCH 2/7] add dropoff percentages to funnel report --- components/messages.js | 1 + components/metrics/DataTable.js | 2 +- components/pages/reports/funnel/FunnelTable.js | 3 +-- lib/prisma.ts | 7 +++++-- queries/analytics/pageview/getPageviewFunnel.ts | 9 +++++++-- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/components/messages.js b/components/messages.js index 32c89687..7bd4e9bc 100644 --- a/components/messages.js +++ b/components/messages.js @@ -161,6 +161,7 @@ export const labels = defineMessages({ overview: { id: 'labels.overview', defaultMessage: 'Overview' }, totalRecords: { id: 'labels.total-records', defaultMessage: 'Total records' }, insights: { id: 'label.insights', defaultMessage: 'Insights' }, + dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' }, }); export const messages = defineMessages({ diff --git a/components/metrics/DataTable.js b/components/metrics/DataTable.js index e2e9462d..9c5fb559 100644 --- a/components/metrics/DataTable.js +++ b/components/metrics/DataTable.js @@ -78,7 +78,7 @@ const AnimatedRow = ({ showPercentage = true, }) => { const props = useSpring({ - width: percent, + width: Number(percent), y: value, from: { width: 0, y: 0 }, config: animate ? config.default : { duration: 0 }, diff --git a/components/pages/reports/funnel/FunnelTable.js b/components/pages/reports/funnel/FunnelTable.js index ff6bdfb5..9ae8ab58 100644 --- a/components/pages/reports/funnel/FunnelTable.js +++ b/components/pages/reports/funnel/FunnelTable.js @@ -6,13 +6,12 @@ import { ReportContext } from '../Report'; export function FunnelTable() { const { report } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); - return ( ); } diff --git a/lib/prisma.ts b/lib/prisma.ts index 2197a1ae..722535c2 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -48,11 +48,11 @@ function getDropoffQuery() { const db = getDatabaseType(process.env.DATABASE_URL); if (db === POSTGRESQL) { - return `round((1.0 - count::numeric/lag(count, 1) over ()),2)`; + return `round((1.0 - count::numeric/lag(nullif(count, 0), 1) over ()),2) * 100`; } if (db === MYSQL) { - return `round((1.0 - count/LAG(count, 1) OVER (ORDER BY level)),2)`; + return `round((1.0 - count/LAG(nullif(count, 0), 1) OVER (ORDER BY level)),2) * 100`; } } @@ -173,6 +173,7 @@ function getFunnelQuery( and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} and we.referrer_path = $${i + initParamLength} and we.url_path = $${levelNumber + initParamLength} + and we.created_at between $2 and $3 and we.website_id = $1${toUuid()} )`; } @@ -216,6 +217,8 @@ async function rawQuery(query: string, params: never[] = []): Promise { const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query; + // console.log(sql); + // console.log(params); return prisma.rawQuery(sql, params); } diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index ef52101f..b7d918b9 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -31,6 +31,7 @@ async function relationalQuery( { x: string; y: number; + z: number; }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; @@ -60,8 +61,12 @@ async function relationalQuery( from levelCount; `, params, - ).then((a: { [key: string]: number }) => { - return urls.map((b, i) => ({ x: b, y: a[0][`level${i + 1}`] || 0 })); + ).then(results => { + return urls.map((a, i) => ({ + x: a, + y: results[i]?.count || 0, + z: results[i]?.drop_off || 0, + })); }); } From 711ad2bbcfdbac4d48230dd7f866f4ea09f245fe Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Mon, 24 Jul 2023 11:57:46 -0700 Subject: [PATCH 3/7] remove dropoff calc from db query --- components/metrics/DataTable.js | 2 +- lib/prisma.ts | 15 --------- .../analytics/pageview/getPageviewFunnel.ts | 32 ++++++++----------- 3 files changed, 15 insertions(+), 34 deletions(-) diff --git a/components/metrics/DataTable.js b/components/metrics/DataTable.js index 9c5fb559..e2e9462d 100644 --- a/components/metrics/DataTable.js +++ b/components/metrics/DataTable.js @@ -78,7 +78,7 @@ const AnimatedRow = ({ showPercentage = true, }) => { const props = useSpring({ - width: Number(percent), + width: percent, y: value, from: { width: 0, y: 0 }, config: animate ? config.default : { duration: 0 }, diff --git a/lib/prisma.ts b/lib/prisma.ts index 722535c2..85dc25b8 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -44,18 +44,6 @@ function getAddMinutesQuery(field: string, minutes: number) { } } -function getDropoffQuery() { - const db = getDatabaseType(process.env.DATABASE_URL); - - if (db === POSTGRESQL) { - return `round((1.0 - count::numeric/lag(nullif(count, 0), 1) over ()),2) * 100`; - } - - if (db === MYSQL) { - return `round((1.0 - count/LAG(nullif(count, 0), 1) OVER (ORDER BY level)),2) * 100`; - } -} - function getDateQuery(field: string, unit: string, timezone?: string): string { const db = getDatabaseType(process.env.DATABASE_URL); @@ -217,15 +205,12 @@ async function rawQuery(query: string, params: never[] = []): Promise { const sql = db === MYSQL ? query.replace(/\$[0-9]+/g, '?') : query; - // console.log(sql); - // console.log(params); return prisma.rawQuery(sql, params); } export default { ...prisma, getAddMinutesQuery, - getDropoffQuery, getDateQuery, getTimestampInterval, getFilterQuery, diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index b7d918b9..422338da 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -35,7 +35,7 @@ async function relationalQuery( }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getFunnelQuery, toUuid, getDropoffQuery } = prisma; + const { rawQuery, getFunnelQuery, toUuid } = prisma; const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes); const params: any = [websiteId, startDate, endDate, ...urls]; @@ -47,25 +47,19 @@ async function relationalQuery( where url_path in (${urlFilterQuery}) and website_id = $1${toUuid()} and created_at between $2 and $3 - ),level1 AS ( - select distinct session_id, created_at - from level0 - where url_path = $4 - )${levelQuery}, levelCount as ( - ${sumQuery} - order by level) - select - level, - count, - ${getDropoffQuery()} as drop_off - from levelCount; - `, + ),level1 AS ( + select distinct session_id, created_at + from level0 + where url_path = $4 + )${levelQuery} + ${sumQuery} + ORDER BY level;`, params, ).then(results => { return urls.map((a, i) => ({ x: a, y: results[i]?.count || 0, - z: results[i]?.drop_off || 0, + z: (1 - (Number(results[i]?.count) * 1.0) / Number(results[i - 1]?.count)) * 100 || 0, // drop off })); }); } @@ -96,7 +90,7 @@ async function clickhouseQuery( return rawQuery<{ level: number; count: number }[]>( ` - select level, + WITH funnel as (select level, count(*) AS count from ( select session_id, @@ -111,13 +105,15 @@ async function clickhouseQuery( group by 1 ) group by level - order by level asc; + order by level asc) + select * from funnel where level > 0; `, params, ).then(results => { return urls.map((a, i) => ({ x: a, - y: results[i + 1]?.count || 0, + y: results[i]?.count || 0, + z: (1 - (Number(results[i]?.count) * 1.0) / Number(results[i - 1]?.count)) * 100 || 0, // drop off })); }); } From 5a0908964c5897733121c758fbe62b41846735fe Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 25 Jul 2023 13:23:44 -0700 Subject: [PATCH 4/7] align CH / postgres funnel queries --- lib/clickhouse.ts | 41 +++++++++++++---- lib/prisma.ts | 4 +- .../analytics/pageview/getPageviewFunnel.ts | 46 ++++++++----------- 3 files changed, 54 insertions(+), 37 deletions(-) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index eb73d83c..d7cc572b 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -121,24 +121,47 @@ function getFilterQuery(filters = {}, params = {}) { return query.join('\n'); } -function getFunnelQuery(urls: string[]): { - columnsQuery: string; - conditionQuery: string; +function getFunnelQuery( + urls: string[], + windowMinutes: number, +): { + levelQuery: string; + sumQuery: string; + urlFilterQuery: string; urlParams: { [key: string]: string }; } { return urls.reduce( (pv, cv, i) => { - pv.columnsQuery += `\n,url_path = {url${i}:String}${ - i > 0 && urls[i - 1] ? ` AND referrer_path = {url${i - 1}:String}` : '' - }`; - pv.conditionQuery += `${i > 0 ? ',' : ''} {url${i}:String}`; + const levelNumber = i + 1; + const startSum = i > 0 ? 'union all ' : ''; + const startFilter = i > 0 ? ', ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += `\n + , level${levelNumber} AS ( + select distinct y.session_id as session_id, + y.url_path as url_path, + y.referrer_path as referrer_path, + y.created_at as created_at + from level${i} x + join level0 y + on x.session_id = y.session_id + where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute + and y.referrer_path = {url${i - 1}:String} + and y.url_path = {url${i}:String} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + pv.urlFilterQuery += `${startFilter}{url${i}:String} `; pv.urlParams[`url${i}`] = cv; return pv; }, { - columnsQuery: '', - conditionQuery: '', + levelQuery: '', + sumQuery: '', + urlFilterQuery: '', urlParams: {}, }, ); diff --git a/lib/prisma.ts b/lib/prisma.ts index 85dc25b8..e250987e 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -153,7 +153,7 @@ function getFunnelQuery( if (levelNumber >= 2) { pv.levelQuery += `\n , level${levelNumber} AS ( - select distinct l.session_id, we.created_at + select distinct we.session_id, we.created_at from level${i} l join website_event we on l.session_id = we.session_id @@ -161,7 +161,7 @@ function getFunnelQuery( and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} and we.referrer_path = $${i + initParamLength} and we.url_path = $${levelNumber + initParamLength} - and we.created_at between $2 and $3 + and we.created_at <= $3 and we.website_id = $1${toUuid()} )`; } diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index 422338da..3af1545a 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -41,17 +41,14 @@ async function relationalQuery( const params: any = [websiteId, startDate, endDate, ...urls]; return rawQuery( - `WITH level0 AS ( - select distinct session_id, url_path, referrer_path, created_at + `WITH level1 AS ( + select distinct session_id, created_at from website_event where url_path in (${urlFilterQuery}) and website_id = $1${toUuid()} and created_at between $2 and $3 - ),level1 AS ( - select distinct session_id, created_at - from level0 - where url_path = $4 - )${levelQuery} + and url_path = $4) + ${levelQuery} ${sumQuery} ORDER BY level;`, params, @@ -76,38 +73,35 @@ async function clickhouseQuery( { x: string; y: number; + z: number; }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; const { rawQuery, getBetweenDates, getFunnelQuery } = clickhouse; - const { columnsQuery, urlParams } = getFunnelQuery(urls); + const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getFunnelQuery(urls, windowMinutes); const params = { websiteId, - window: windowMinutes * 60, ...urlParams, }; return rawQuery<{ level: number; count: number }[]>( ` - WITH funnel as (select level, - count(*) AS count + WITH level0 AS ( + select distinct session_id, url_path, referrer_path, created_at + from umami.website_event + where url_path in (${urlFilterQuery}) + and website_id = {websiteId:UUID} + and ${getBetweenDates('created_at', startDate, endDate)} + ), level1 AS ( + select * + from level0 + where url_path = {url0:String}) + ${levelQuery} + select * from ( - select session_id, - windowFunnel({window:UInt32}, 'strict_increase') - ( - created_at - ${columnsQuery} - ) AS level - from website_event - where website_id = {websiteId:UUID} - and ${getBetweenDates('created_at', startDate, endDate)} - group by 1 - ) - group by level - order by level asc) - select * from funnel where level > 0; - `, + ${sumQuery} + ) ORDER BY level;`, params, ).then(results => { return urls.map((a, i) => ({ From 70367ffcc4e1cdde92d240967e7a7078c23d1459 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Tue, 25 Jul 2023 14:54:56 -0700 Subject: [PATCH 5/7] build mysql funnel params --- lib/prisma.ts | 65 ++++++++++++------- .../analytics/pageview/getPageviewFunnel.ts | 12 ++-- 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/lib/prisma.ts b/lib/prisma.ts index e250987e..5177acc2 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -32,7 +32,7 @@ function toUuid(): string { } } -function getAddMinutesQuery(field: string, minutes: number) { +function getAddMinutesQuery(field: string, minutes: number): string { const db = getDatabaseType(process.env.DATABASE_URL); if (db === POSTGRESQL) { @@ -134,13 +134,49 @@ function getFilterQuery(filters = {}, params = []): string { return query.join('\n'); } +function parseFilters( + filters: { [key: string]: any } = {}, + params = [], + sessionKey = 'session_id', +) { + const { os, browser, device, country, region, city } = filters; + + return { + joinSession: + os || browser || device || country || region || city + ? `inner join session on website_event.${sessionKey} = session.${sessionKey}` + : '', + filterQuery: getFilterQuery(filters, params), + }; +} + +function getFunnelParams(endDate: Date, websiteId: string, urls: string[]): any[] { + const db = getDatabaseType(process.env.DATABASE_URL); + + if (db === POSTGRESQL) { + return urls; + } + + if (db === MYSQL) { + let params = []; + params.push(urls[0]); + for (let i = 0; i < urls.length - 1; i++) { + params = params.concat([urls[i], urls[i + 1], endDate, websiteId]); + } + + return params; + } +} + function getFunnelQuery( urls: string[], + endDate: Date, + websiteId: string, windowMinutes: number, ): { levelQuery: string; sumQuery: string; - urlFilterQuery: string; + urlParams: any[]; } { const initParamLength = 3; @@ -148,7 +184,6 @@ function getFunnelQuery( (pv, cv, i) => { const levelNumber = i + 1; const startSum = i > 0 ? 'union ' : ''; - const startFilter = i > 0 ? ', ' : ''; if (levelNumber >= 2) { pv.levelQuery += `\n @@ -167,35 +202,18 @@ function getFunnelQuery( } pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - - pv.urlFilterQuery += `${startFilter}$${levelNumber + initParamLength} `; + pv.urlParams = getFunnelParams(endDate, websiteId, urls); return pv; }, { levelQuery: '', sumQuery: '', - urlFilterQuery: '', + urlParams: [], }, ); } -function parseFilters( - filters: { [key: string]: any } = {}, - params = [], - sessionKey = 'session_id', -) { - const { os, browser, device, country, region, city } = filters; - - return { - joinSession: - os || browser || device || country || region || city - ? `inner join session on website_event.${sessionKey} = session.${sessionKey}` - : '', - filterQuery: getFilterQuery(filters, params), - }; -} - async function rawQuery(query: string, params: never[] = []): Promise { const db = getDatabaseType(process.env.DATABASE_URL); @@ -214,9 +232,10 @@ export default { getDateQuery, getTimestampInterval, getFilterQuery, - getFunnelQuery, getEventDataFilterQuery, toUuid, parseFilters, + getFunnelParams, + getFunnelQuery, rawQuery, }; diff --git a/queries/analytics/pageview/getPageviewFunnel.ts b/queries/analytics/pageview/getPageviewFunnel.ts index 3af1545a..bdfb1c5b 100644 --- a/queries/analytics/pageview/getPageviewFunnel.ts +++ b/queries/analytics/pageview/getPageviewFunnel.ts @@ -36,16 +36,20 @@ async function relationalQuery( > { const { windowMinutes, startDate, endDate, urls } = criteria; const { rawQuery, getFunnelQuery, toUuid } = prisma; - const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes); + const { levelQuery, sumQuery, urlParams } = getFunnelQuery( + urls, + endDate, + websiteId, + windowMinutes, + ); - const params: any = [websiteId, startDate, endDate, ...urls]; + const params: any = [websiteId, startDate, endDate, ...urlParams]; return rawQuery( `WITH level1 AS ( select distinct session_id, created_at from website_event - where url_path in (${urlFilterQuery}) - and website_id = $1${toUuid()} + where website_id = $1${toUuid()} and created_at between $2 and $3 and url_path = $4) ${levelQuery} From cdf745b6b431aee5fa0785a60151efc7bdac3a35 Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Thu, 27 Jul 2023 15:23:20 -0700 Subject: [PATCH 6/7] update funnel dynamic built query --- lib/prisma.ts | 30 ++------------------------ queries/analytics/reports/getFunnel.ts | 15 ++++--------- 2 files changed, 6 insertions(+), 39 deletions(-) diff --git a/lib/prisma.ts b/lib/prisma.ts index 2aa2193c..06b42acf 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -108,24 +108,6 @@ function parseFilters( }; } -function getFunnelParams(endDate: Date, websiteId: string, urls: string[]): any[] { - const db = getDatabaseType(process.env.DATABASE_URL); - - if (db === POSTGRESQL) { - return urls; - } - - if (db === MYSQL) { - let params = []; - params.push(urls[0]); - for (let i = 0; i < urls.length - 1; i++) { - params = params.concat([urls[i], urls[i + 1], endDate, websiteId]); - } - - return params; - } -} - function getFunnelQuery( urls: string[], endDate: Date, @@ -134,10 +116,7 @@ function getFunnelQuery( ): { levelQuery: string; sumQuery: string; - urlParams: any[]; } { - const initParamLength = 3; - return urls.reduce( (pv, cv, i) => { const levelNumber = i + 1; @@ -152,22 +131,20 @@ function getFunnelQuery( on l.session_id = we.session_id where we.created_at between l.created_at and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} - and we.referrer_path = $${i + initParamLength} - and we.url_path = $${levelNumber + initParamLength} + and we.referrer_path = {{${i - 1}}} + and we.url_path = {{${i}}} and we.created_at <= {{endDate}} and we.website_id = {{websiteId}}${toUuid()} )`; } pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - pv.urlParams = getFunnelParams(endDate, websiteId, urls); return pv; }, { levelQuery: '', sumQuery: '', - urlParams: [], }, ); } @@ -179,10 +156,8 @@ async function rawQuery(sql: string, data: object): Promise { if (db !== POSTGRESQL && db !== MYSQL) { return Promise.reject(new Error('Unknown database.')); } - const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => { const [, name, type] = args; - params.push(data[name]); return db === MYSQL ? '?' : `$${params.length}${type ?? ''}`; @@ -199,7 +174,6 @@ export default { getFilterQuery, toUuid, parseFilters, - getFunnelParams, getFunnelQuery, rawQuery, }; diff --git a/queries/analytics/reports/getFunnel.ts b/queries/analytics/reports/getFunnel.ts index 9e1fd123..e1718e84 100644 --- a/queries/analytics/reports/getFunnel.ts +++ b/queries/analytics/reports/getFunnel.ts @@ -36,26 +36,19 @@ async function relationalQuery( > { const { windowMinutes, startDate, endDate, urls } = criteria; const { rawQuery, getFunnelQuery, toUuid } = prisma; - const { levelQuery, sumQuery, urlParams } = getFunnelQuery( - urls, - endDate, - websiteId, - windowMinutes, - ); - - const params: any = [websiteId, startDate, endDate, ...urlParams]; + const { levelQuery, sumQuery } = getFunnelQuery(urls, endDate, websiteId, windowMinutes); return rawQuery( `WITH level1 AS ( select distinct session_id, created_at from website_event where website_id = {{websiteId}}${toUuid()} - and created_at between {{startDate}} and {{endDate}} - and url_path = $4) + and created_at between {{startDate}} and {{endDate}} + and url_path = {{0}}) ${levelQuery} ${sumQuery} ORDER BY level;`, - params, + { websiteId, startDate, endDate, ...urls }, ).then(results => { return urls.map((a, i) => ({ x: a, From aae840ae337e5f70e2dfb8aeae0b54732bea94ee Mon Sep 17 00:00:00 2001 From: Francis Cao Date: Fri, 28 Jul 2023 10:47:14 -0700 Subject: [PATCH 7/7] resolving pr comments --- lib/clickhouse.ts | 47 ----------- lib/prisma.ts | 55 ------------ queries/analytics/reports/getFunnel.ts | 112 ++++++++++++++++++++++--- 3 files changed, 100 insertions(+), 114 deletions(-) diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index 959630ef..b3dc2c48 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -77,52 +77,6 @@ function getFilterQuery(filters = {}, params = {}) { return query.join('\n'); } -function getFunnelQuery( - urls: string[], - windowMinutes: number, -): { - levelQuery: string; - sumQuery: string; - urlFilterQuery: string; - urlParams: { [key: string]: string }; -} { - return urls.reduce( - (pv, cv, i) => { - const levelNumber = i + 1; - const startSum = i > 0 ? 'union all ' : ''; - const startFilter = i > 0 ? ', ' : ''; - - if (levelNumber >= 2) { - pv.levelQuery += `\n - , level${levelNumber} AS ( - select distinct y.session_id as session_id, - y.url_path as url_path, - y.referrer_path as referrer_path, - y.created_at as created_at - from level${i} x - join level0 y - on x.session_id = y.session_id - where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute - and y.referrer_path = {url${i - 1}:String} - and y.url_path = {url${i}:String} - )`; - } - - pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - pv.urlFilterQuery += `${startFilter}{url${i}:String} `; - pv.urlParams[`url${i}`] = cv; - - return pv; - }, - { - levelQuery: '', - sumQuery: '', - urlFilterQuery: '', - urlParams: {}, - }, - ); -} - function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) { return { filterQuery: getFilterQuery(filters, params), @@ -169,7 +123,6 @@ export default { getDateQuery, getDateFormat, getFilterQuery, - getFunnelQuery, parseFilters, findUnique, findFirst, diff --git a/lib/prisma.ts b/lib/prisma.ts index 06b42acf..9b02b31b 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -19,18 +19,6 @@ const POSTGRESQL_DATE_FORMATS = { year: 'YYYY-01-01', }; -function toUuid(): string { - const db = getDatabaseType(process.env.DATABASE_URL); - - if (db === POSTGRESQL) { - return '::uuid'; - } - - if (db === MYSQL) { - return ''; - } -} - function getAddMinutesQuery(field: string, minutes: number): string { const db = getDatabaseType(process.env.DATABASE_URL); @@ -108,47 +96,6 @@ function parseFilters( }; } -function getFunnelQuery( - urls: string[], - endDate: Date, - websiteId: string, - windowMinutes: number, -): { - levelQuery: string; - sumQuery: string; -} { - return urls.reduce( - (pv, cv, i) => { - const levelNumber = i + 1; - const startSum = i > 0 ? 'union ' : ''; - - if (levelNumber >= 2) { - pv.levelQuery += `\n - , level${levelNumber} AS ( - select distinct we.session_id, we.created_at - from level${i} l - join website_event we - on l.session_id = we.session_id - where we.created_at between l.created_at - and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} - and we.referrer_path = {{${i - 1}}} - and we.url_path = {{${i}}} - and we.created_at <= {{endDate}} - and we.website_id = {{websiteId}}${toUuid()} - )`; - } - - pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; - - return pv; - }, - { - levelQuery: '', - sumQuery: '', - }, - ); -} - async function rawQuery(sql: string, data: object): Promise { const db = getDatabaseType(); const params = []; @@ -172,8 +119,6 @@ export default { getDateQuery, getTimestampIntervalQuery, getFilterQuery, - toUuid, parseFilters, - getFunnelQuery, rawQuery, }; diff --git a/queries/analytics/reports/getFunnel.ts b/queries/analytics/reports/getFunnel.ts index e1718e84..fcaa9307 100644 --- a/queries/analytics/reports/getFunnel.ts +++ b/queries/analytics/reports/getFunnel.ts @@ -35,20 +35,64 @@ async function relationalQuery( }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getFunnelQuery, toUuid } = prisma; - const { levelQuery, sumQuery } = getFunnelQuery(urls, endDate, websiteId, windowMinutes); + const { rawQuery, getAddMinutesQuery } = prisma; + const { levelQuery, sumQuery } = getFunnelQuery(urls, windowMinutes); + + function getFunnelQuery( + urls: string[], + windowMinutes: number, + ): { + levelQuery: string; + sumQuery: string; + } { + return urls.reduce( + (pv, cv, i) => { + const levelNumber = i + 1; + const startSum = i > 0 ? 'union ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += ` + , level${levelNumber} AS ( + select distinct we.session_id, we.created_at + from level${i} l + join website_event we + on l.session_id = we.session_id + where we.created_at between l.created_at + and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} + and we.referrer_path = {{${i - 1}}} + and we.url_path = {{${i}}} + and we.created_at <= {{endDate}} + and we.website_id = {{websiteId::uuid}} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + + return pv; + }, + { + levelQuery: '', + sumQuery: '', + }, + ); + } return rawQuery( `WITH level1 AS ( select distinct session_id, created_at from website_event - where website_id = {{websiteId}}${toUuid()} + where website_id = {{websiteId::uuid}} and created_at between {{startDate}} and {{endDate}} and url_path = {{0}}) ${levelQuery} ${sumQuery} ORDER BY level;`, - { websiteId, startDate, endDate, ...urls }, + { + websiteId, + startDate, + endDate, + ...urls, + }, ).then(results => { return urls.map((a, i) => ({ x: a, @@ -74,15 +118,54 @@ async function clickhouseQuery( }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getFunnelQuery } = clickhouse; + const { rawQuery } = clickhouse; const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getFunnelQuery(urls, windowMinutes); - const params = { - websiteId, - startDate, - endDate, - ...urlParams, - }; + function getFunnelQuery( + urls: string[], + windowMinutes: number, + ): { + levelQuery: string; + sumQuery: string; + urlFilterQuery: string; + urlParams: { [key: string]: string }; + } { + return urls.reduce( + (pv, cv, i) => { + const levelNumber = i + 1; + const startSum = i > 0 ? 'union all ' : ''; + const startFilter = i > 0 ? ', ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += `\n + , level${levelNumber} AS ( + select distinct y.session_id as session_id, + y.url_path as url_path, + y.referrer_path as referrer_path, + y.created_at as created_at + from level${i} x + join level0 y + on x.session_id = y.session_id + where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute + and y.referrer_path = {url${i - 1}:String} + and y.url_path = {url${i}:String} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + pv.urlFilterQuery += `${startFilter}{url${i}:String} `; + pv.urlParams[`url${i}`] = cv; + + return pv; + }, + { + levelQuery: '', + sumQuery: '', + urlFilterQuery: '', + urlParams: {}, + }, + ); + } return rawQuery<{ level: number; count: number }[]>( ` @@ -101,7 +184,12 @@ async function clickhouseQuery( from ( ${sumQuery} ) ORDER BY level;`, - params, + { + websiteId, + startDate, + endDate, + ...urlParams, + }, ).then(results => { return urls.map((a, i) => ({ x: a,