diff --git a/components/messages.js b/components/messages.js index 32c89687..7bd4e9bc 100644 --- a/components/messages.js +++ b/components/messages.js @@ -161,6 +161,7 @@ export const labels = defineMessages({ overview: { id: 'labels.overview', defaultMessage: 'Overview' }, totalRecords: { id: 'labels.total-records', defaultMessage: 'Total records' }, insights: { id: 'label.insights', defaultMessage: 'Insights' }, + dropoff: { id: 'label.dropoff', defaultMessage: 'Dropoff' }, }); export const messages = defineMessages({ diff --git a/components/pages/reports/funnel/FunnelTable.js b/components/pages/reports/funnel/FunnelTable.js index ff6bdfb5..9ae8ab58 100644 --- a/components/pages/reports/funnel/FunnelTable.js +++ b/components/pages/reports/funnel/FunnelTable.js @@ -6,13 +6,12 @@ import { ReportContext } from '../Report'; export function FunnelTable() { const { report } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); - return ( ); } diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index 3684d075..b3dc2c48 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -77,29 +77,6 @@ function getFilterQuery(filters = {}, params = {}) { return query.join('\n'); } -function getFunnelQuery(urls: string[]): { - columnsQuery: string; - conditionQuery: string; - urlParams: { [key: string]: string }; -} { - return urls.reduce( - (pv, cv, i) => { - pv.columnsQuery += `\n,url_path = {url${i}:String}${ - i > 0 && urls[i - 1] ? ` AND referrer_path = {url${i - 1}:String}` : '' - }`; - pv.conditionQuery += `${i > 0 ? ',' : ''} {url${i}:String}`; - pv.urlParams[`url${i}`] = cv; - - return pv; - }, - { - columnsQuery: '', - conditionQuery: '', - urlParams: {}, - }, - ); -} - function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) { return { filterQuery: getFilterQuery(filters, params), @@ -146,7 +123,6 @@ export default { getDateQuery, getDateFormat, getFilterQuery, - getFunnelQuery, parseFilters, findUnique, findFirst, diff --git a/lib/prisma.ts b/lib/prisma.ts index b02f69f7..9b02b31b 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -19,8 +19,8 @@ const POSTGRESQL_DATE_FORMATS = { year: 'YYYY-01-01', }; -function getAddMinutesQuery(field: string, minutes: number) { - const db = getDatabaseType(); +function getAddMinutesQuery(field: string, minutes: number): string { + const db = getDatabaseType(process.env.DATABASE_URL); if (db === POSTGRESQL) { return `${field} + interval '${minutes} minute'`; @@ -80,53 +80,6 @@ function getFilterQuery(filters = {}, params = []): string { return query.join('\n'); } -function getFunnelQuery( - urls: string[], - windowMinutes: number, -): { - levelQuery: string; - sumQuery: string; - urlFilterQuery: string; -} { - const initParamLength = 3; - - return urls.reduce( - (pv, cv, i) => { - const levelNumber = i + 1; - const start = i > 0 ? ',' : ''; - - if (levelNumber >= 2) { - pv.levelQuery += `\n - , level${levelNumber} AS ( - select cl.*, - l0.created_at level_${levelNumber}_created_at, - l0.url_path as level_${levelNumber}_url - from level${i} cl - left join website_event l0 - on cl.session_id = l0.session_id - and l0.created_at between cl.level_${i}_created_at - and ${getAddMinutesQuery(`cl.level_${i}_created_at`, windowMinutes)} - and l0.referrer_path = $${i + initParamLength} - and l0.url_path = $${levelNumber + initParamLength} - and created_at between $2 and $3 - and website_id = $1 - )`; - } - - pv.sumQuery += `\n${start}SUM(CASE WHEN level_${levelNumber}_url is not null THEN 1 ELSE 0 END) AS level${levelNumber}`; - - pv.urlFilterQuery += `\n${start}$${levelNumber + initParamLength} `; - - return pv; - }, - { - levelQuery: '', - sumQuery: '', - urlFilterQuery: '', - }, - ); -} - function parseFilters( filters: { [key: string]: any } = {}, params = [], @@ -150,10 +103,8 @@ async function rawQuery(sql: string, data: object): Promise { if (db !== POSTGRESQL && db !== MYSQL) { return Promise.reject(new Error('Unknown database.')); } - const query = sql?.replaceAll(/\{\{\s*(\w+)(::\w+)?\s*}}/g, (...args) => { const [, name, type] = args; - params.push(data[name]); return db === MYSQL ? '?' : `$${params.length}${type ?? ''}`; @@ -168,7 +119,6 @@ export default { getDateQuery, getTimestampIntervalQuery, getFilterQuery, - getFunnelQuery, parseFilters, rawQuery, }; diff --git a/queries/analytics/reports/getFunnel.ts b/queries/analytics/reports/getFunnel.ts index 1dde1a13..fcaa9307 100644 --- a/queries/analytics/reports/getFunnel.ts +++ b/queries/analytics/reports/getFunnel.ts @@ -31,31 +31,74 @@ async function relationalQuery( { x: string; y: number; + z: number; }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getFunnelQuery } = prisma; - const { levelQuery, sumQuery, urlFilterQuery } = getFunnelQuery(urls, windowMinutes); + const { rawQuery, getAddMinutesQuery } = prisma; + const { levelQuery, sumQuery } = getFunnelQuery(urls, windowMinutes); + + function getFunnelQuery( + urls: string[], + windowMinutes: number, + ): { + levelQuery: string; + sumQuery: string; + } { + return urls.reduce( + (pv, cv, i) => { + const levelNumber = i + 1; + const startSum = i > 0 ? 'union ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += ` + , level${levelNumber} AS ( + select distinct we.session_id, we.created_at + from level${i} l + join website_event we + on l.session_id = we.session_id + where we.created_at between l.created_at + and ${getAddMinutesQuery(`l.created_at `, windowMinutes)} + and we.referrer_path = {{${i - 1}}} + and we.url_path = {{${i}}} + and we.created_at <= {{endDate}} + and we.website_id = {{websiteId::uuid}} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + + return pv; + }, + { + levelQuery: '', + sumQuery: '', + }, + ); + } return rawQuery( - `WITH level0 AS ( - select distinct session_id, url_path, referrer_path, created_at + `WITH level1 AS ( + select distinct session_id, created_at from website_event - where url_path in (${urlFilterQuery}) - and website_id = {{websiteId::uuid}} - and created_at between {{startDate}} and {{endDate}} - ),level1 AS ( - select distinct session_id, url_path as level_1_url, created_at as level_1_created_at - from level0 - where url_path = $4 - )${levelQuery} - - SELECT ${sumQuery} - from level${urls.length}; - `, - { websiteId, startDate, endDate, ...urls }, - ).then((a: { [key: string]: number }) => { - return urls.map((b, i) => ({ x: b, y: a[0][`level${i + 1}`] || 0 })); + where website_id = {{websiteId::uuid}} + and created_at between {{startDate}} and {{endDate}} + and url_path = {{0}}) + ${levelQuery} + ${sumQuery} + ORDER BY level;`, + { + websiteId, + startDate, + endDate, + ...urls, + }, + ).then(results => { + return urls.map((a, i) => ({ + x: a, + y: results[i]?.count || 0, + z: (1 - (Number(results[i]?.count) * 1.0) / Number(results[i - 1]?.count)) * 100 || 0, // drop off + })); }); } @@ -71,42 +114,87 @@ async function clickhouseQuery( { x: string; y: number; + z: number; }[] > { const { windowMinutes, startDate, endDate, urls } = criteria; - const { rawQuery, getFunnelQuery } = clickhouse; - const { columnsQuery, urlParams } = getFunnelQuery(urls); + const { rawQuery } = clickhouse; + const { levelQuery, sumQuery, urlFilterQuery, urlParams } = getFunnelQuery(urls, windowMinutes); + + function getFunnelQuery( + urls: string[], + windowMinutes: number, + ): { + levelQuery: string; + sumQuery: string; + urlFilterQuery: string; + urlParams: { [key: string]: string }; + } { + return urls.reduce( + (pv, cv, i) => { + const levelNumber = i + 1; + const startSum = i > 0 ? 'union all ' : ''; + const startFilter = i > 0 ? ', ' : ''; + + if (levelNumber >= 2) { + pv.levelQuery += `\n + , level${levelNumber} AS ( + select distinct y.session_id as session_id, + y.url_path as url_path, + y.referrer_path as referrer_path, + y.created_at as created_at + from level${i} x + join level0 y + on x.session_id = y.session_id + where y.created_at between x.created_at and x.created_at + interval ${windowMinutes} minute + and y.referrer_path = {url${i - 1}:String} + and y.url_path = {url${i}:String} + )`; + } + + pv.sumQuery += `\n${startSum}select ${levelNumber} as level, count(distinct(session_id)) as count from level${levelNumber}`; + pv.urlFilterQuery += `${startFilter}{url${i}:String} `; + pv.urlParams[`url${i}`] = cv; + + return pv; + }, + { + levelQuery: '', + sumQuery: '', + urlFilterQuery: '', + urlParams: {}, + }, + ); + } return rawQuery<{ level: number; count: number }[]>( ` - SELECT level, - count(*) AS count - FROM ( - SELECT session_id, - windowFunnel({window:UInt32}, 'strict_increase') - ( - created_at - ${columnsQuery} - ) AS level - FROM website_event - WHERE website_id = {websiteId:UUID} - AND created_at BETWEEN {startDate:DateTime} AND {endDate:DateTime} - GROUP BY 1 - ) - GROUP BY level - ORDER BY level ASC; - `, + WITH level0 AS ( + select distinct session_id, url_path, referrer_path, created_at + from umami.website_event + where url_path in (${urlFilterQuery}) + and website_id = {websiteId:UUID} + and created_at between {startDate:DateTime64} and {endDate:DateTime64} + ), level1 AS ( + select * + from level0 + where url_path = {url0:String}) + ${levelQuery} + select * + from ( + ${sumQuery} + ) ORDER BY level;`, { websiteId, startDate, endDate, - window: windowMinutes * 60, ...urlParams, }, ).then(results => { return urls.map((a, i) => ({ x: a, - y: results[i + 1]?.count || 0, + y: results[i]?.count || 0, + z: (1 - (Number(results[i]?.count) * 1.0) / Number(results[i - 1]?.count)) * 100 || 0, // drop off })); }); }