From d81a7fec999ce681185645f7baf929ce0baae638 Mon Sep 17 00:00:00 2001 From: Mike Cao Date: Thu, 6 Feb 2025 20:39:33 -0800 Subject: [PATCH] Added channel metrics. --- .../api/websites/[websiteId]/metrics/route.ts | 128 +++++++++++++++++- src/queries/sql/getChannelMetrics.ts | 47 +++---- 2 files changed, 147 insertions(+), 28 deletions(-) diff --git a/src/app/api/websites/[websiteId]/metrics/route.ts b/src/app/api/websites/[websiteId]/metrics/route.ts index 50bfc387..e713bdf3 100644 --- a/src/app/api/websites/[websiteId]/metrics/route.ts +++ b/src/app/api/websites/[websiteId]/metrics/route.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import thenby from 'thenby'; import { canViewWebsite } from '@/lib/auth'; import { SESSION_COLUMNS, EVENT_COLUMNS, FILTER_COLUMNS, OPERATORS } from '@/lib/constants'; import { getRequestFilters, getRequestDateRange, parseRequest } from '@/lib/request'; @@ -81,8 +82,133 @@ export async function GET( if (type === 'channel') { const data = await getChannelMetrics(websiteId, filters); - return json(data); + const channels = getChannels(data); + + return json( + Object.keys(channels) + .map(key => ({ x: key, y: channels[key] })) + .sort(thenby.firstBy('y', -1)), + ); } return badRequest(); } + +const SOCIAL_DOMAINS = [ + 'facebook.com', + 'fb.com', + 'instagram.com', + 'ig.com', + 'twitter.com', + 't.co', + 'x.com', + 'linkedin.', + 'tiktok.', + 'reddit.', + 'threads.net', + 'bsky.app', + 'news.ycombinator.com', +]; + +const SEARCH_DOMAINS = [ + 'google.', + 'bing.com', + 'msn.com', + 'duckduckgo.com', + 'search.brave.com', + 'yandex.', + 'baidu.com', + 'ecosia.org', + 'chatgpt.com', + 'perplexity.ai', +]; + +const SHOPPING_DOMAINS = [ + 'amazon.', + 'ebay.com', + 'walmart.com', + 'alibab.com', + 'aliexpress.com', + 'etsy.com', + 'bestbuy.com', + 'target.com', + 'newegg.com', +]; + +const EMAIL_DOMAINS = ['gmail.', 'mail.yahoo.', 'outlook.', 'hotmail.', 'protonmail.', 'proton.me']; + +const VIDEO_DOMAINS = ['youtube.', 'twitch.']; + +const PAID_AD_PARAMS = [ + 'utm_source=google', + 'gclid=', + 'fbclid=', + 'msclkid=', + 'dclid=', + 'twclid=', + 'li_fat_id=', + 'epik=', + 'ttclid=', + 'scid=', +]; + +function getChannels(data: { domain: string; query: string; visitors: number }[]) { + const channels = { + direct: 0, + referral: 0, + affiliate: 0, + sms: 0, + organic_search: 0, + organic_social: 0, + organic_email: 0, + organic_shopping: 0, + organic_video: 0, + paid_ads: 0, + paid_search: 0, + paid_social: 0, + paid_shopping: 0, + paid_video: 0, + }; + + const match = (value: string) => { + return (str: string | RegExp) => { + return typeof str === 'string' ? value.includes(str) : (str as RegExp).test(value); + }; + }; + + for (const { domain, query, visitors } of data) { + if (!domain && !query) { + channels.direct += visitors; + } + + const prefix = /utm_medium=(.*cp.*|ppc|retargeting|paid.*)/.test(query) ? 'paid' : 'organic'; + + if (SEARCH_DOMAINS.some(match(domain)) || /utm_medium=organic/.test(query)) { + channels[`${prefix}_search`] += visitors; + } else if ( + SOCIAL_DOMAINS.some(match(domain)) || + /utm_medium=(social|social-network|social-media|sm|social network|social media)/.test(query) + ) { + channels[`${prefix}_social`] += visitors; + } else if (EMAIL_DOMAINS.some(match(domain)) || /utm_medium=(.*e[-_ ]?mail.*)/.test(query)) { + channels.organic_email += visitors; + } else if ( + SHOPPING_DOMAINS.some(match(domain)) || + /utm_campaign=(.*(([^a-df-z]|^)shop|shopping).*)/.test(query) + ) { + channels[`${prefix}_shopping`] += visitors; + } else if (VIDEO_DOMAINS.some(match(domain)) || /utm_medium=(.*video.*)/.test(query)) { + channels[`${prefix}_video`] += visitors; + } else if (PAID_AD_PARAMS.some(match(query))) { + channels.paid_ads += visitors; + } else if (/utm_medium=(referral|app|link)/.test(query)) { + channels.referral += visitors; + } else if (/utm_medium=affiliate/.test(query)) { + channels.affiliate += visitors; + } else if (/utm_(source|medium)=sms/.test(query)) { + channels.sms += visitors; + } + } + + return channels; +} diff --git a/src/queries/sql/getChannelMetrics.ts b/src/queries/sql/getChannelMetrics.ts index 084b7a2c..22aa00bb 100644 --- a/src/queries/sql/getChannelMetrics.ts +++ b/src/queries/sql/getChannelMetrics.ts @@ -17,23 +17,15 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { return rawQuery( ` select - website_event.session_id as "sessionId", - website_event.event_name as "eventName", - website_event.created_at as "createdAt", - session.browser, - session.os, - session.device, - session.country, - website_event.url_path as "urlPath", - website_event.referrer_domain as "referrerDomain" + referrer_domain as domain, + referrer_query as query, + count(*) as visitors from website_event - inner join session - on session.session_id = website_event.session_id - where website_event.website_id = {{websiteId::uuid}} - ${filterQuery} - ${dateQuery} - order by website_event.created_at desc - limit 100 + where website_id = {websiteId:UUID} + ${filterQuery} + ${dateQuery} + group by 1, 2 + order by visitors desc `, params, ); @@ -47,20 +39,21 @@ async function clickhouseQuery( const { params, filterQuery, dateQuery } = await parseFilters(websiteId, filters); const sql = ` - select - referrer_domain as x, - count(*) as y - from website_event - where website_id = {websiteId:UUID} - ${filterQuery} - ${dateQuery} - group by 1 - order by y desc - `; + select + referrer_domain as domain, + referrer_query as query, + uniq(session_id) as visitors + from website_event + where website_id = {websiteId:UUID} + ${filterQuery} + ${dateQuery} + group by 1, 2 + order by visitors desc + `; return rawQuery(sql, params).then(a => { return Object.values(a).map(a => { - return { x: a.x, y: Number(a.y) }; + return { ...a, visitors: Number(a.visitors) }; }); }); }