diff --git a/db/clickhouse/schema.sql b/db/clickhouse/schema.sql index 41b3a1b3..be277743 100644 --- a/db/clickhouse/schema.sql +++ b/db/clickhouse/schema.sql @@ -57,7 +57,7 @@ CREATE TABLE umami.event_data event_name String, data_key String, string_value Nullable(String), - number_value Nullable(Decimal64(4)), + number_value Nullable(Decimal64(22, 4)), date_value Nullable(DateTime('UTC')), data_type UInt32, created_at DateTime('UTC'), @@ -73,7 +73,7 @@ CREATE TABLE umami.session_data session_id UUID, data_key String, string_value Nullable(String), - number_value Nullable(Decimal64(4)), + number_value Nullable(Decimal64(22, 4)), date_value Nullable(DateTime('UTC')), data_type UInt32, distinct_id String, diff --git a/src/app/api/reports/utm/route.ts b/src/app/api/reports/utm/route.ts index 17c5c1c6..150773f7 100644 --- a/src/app/api/reports/utm/route.ts +++ b/src/app/api/reports/utm/route.ts @@ -17,8 +17,8 @@ export async function POST(request: Request) { return unauthorized(); } - const parameters = await setWebsiteDate(websiteId, body.parameters); const filters = await getQueryFilters(body.filters, websiteId); + const parameters = await setWebsiteDate(websiteId, body.parameters); const data = await getUTM(websiteId, parameters as UTMParameters, filters); diff --git a/src/app/api/websites/[websiteId]/sessions/route.ts b/src/app/api/websites/[websiteId]/sessions/route.ts index bcc4b8bc..5f81e14c 100644 --- a/src/app/api/websites/[websiteId]/sessions/route.ts +++ b/src/app/api/websites/[websiteId]/sessions/route.ts @@ -2,7 +2,7 @@ import { z } from 'zod'; import { getQueryFilters, parseRequest } from '@/lib/request'; import { unauthorized, json } from '@/lib/response'; import { canViewWebsite } from '@/lib/auth'; -import { dateRangeParams, filterParams, pagingParams } from '@/lib/schema'; +import { dateRangeParams, filterParams, pagingParams, searchParams } from '@/lib/schema'; import { getWebsiteSessions } from '@/queries'; export async function GET( @@ -13,6 +13,7 @@ export async function GET( ...dateRangeParams, ...filterParams, ...pagingParams, + ...searchParams, }); const { auth, query, error } = await parseRequest(request, schema); diff --git a/src/components/metrics/ListExpandedTable.tsx b/src/components/metrics/ListExpandedTable.tsx index dc27b17f..82285237 100644 --- a/src/components/metrics/ListExpandedTable.tsx +++ b/src/components/metrics/ListExpandedTable.tsx @@ -7,11 +7,10 @@ import styles from './ListExpandedTable.module.css'; export interface ListExpandedTableProps { data?: any[]; title?: string; - type?: string; renderLabel?: (row: any, index: number) => ReactNode; } -export function ListExpandedTable({ data = [], title, type, renderLabel }: ListExpandedTableProps) { +export function ListExpandedTable({ data = [], title, renderLabel }: ListExpandedTableProps) { const { formatMessage, labels } = useMessages(); return ( @@ -32,26 +31,18 @@ export function ListExpandedTable({ data = [], title, type, renderLabel }: ListE {row => row?.['pageviews']?.toLocaleString()} - {type !== 'exit' && type !== 'entry' ? ( - - {row => { - const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; - return Math.round(+n) + '%'; - }} - - ) : ( - <> - )} - {type !== 'exit' && type !== 'entry' ? ( - - {row => { - const n = (row?.['totaltime'] / row?.['visits']) * 100; - return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; - }} - - ) : ( - <> - )} + + {row => { + const n = (Math.min(row?.['visits'], row?.['bounces']) / row?.['visits']) * 100; + return Math.round(+n) + '%'; + }} + + + {row => { + const n = (row?.['totaltime'] / row?.['visits']) * 100; + return `${+n < 0 ? '-' : ''}${formatShortTime(Math.abs(~~n), ['m', 's'], ' ')}`; + }} + ); } diff --git a/src/components/metrics/MetricsTable.tsx b/src/components/metrics/MetricsTable.tsx index fb97f8db..c18e46e4 100644 --- a/src/components/metrics/MetricsTable.tsx +++ b/src/components/metrics/MetricsTable.tsx @@ -114,7 +114,7 @@ export function MetricsTable({ return ( - + {allowSearch && } @@ -124,7 +124,7 @@ export function MetricsTable({ {data && (expanded ? ( - + ) : ( ))} diff --git a/src/components/metrics/ReferrersTable.tsx b/src/components/metrics/ReferrersTable.tsx index 0b3f1ee3..718f13ee 100644 --- a/src/components/metrics/ReferrersTable.tsx +++ b/src/components/metrics/ReferrersTable.tsx @@ -1,11 +1,11 @@ -import { Row } from '@umami/react-zen'; -import { FilterLink } from '@/components/common/FilterLink'; import { Favicon } from '@/components/common/Favicon'; import { FilterButtons } from '@/components/common/FilterButtons'; +import { FilterLink } from '@/components/common/FilterLink'; import { useMessages, useNavigation } from '@/components/hooks'; -import { MetricsTable, MetricsTableProps } from './MetricsTable'; -import thenby from 'thenby'; import { GROUPED_DOMAINS } from '@/lib/constants'; +import { emptyFilter } from '@/lib/filters'; +import { Row } from '@umami/react-zen'; +import { MetricsTable, MetricsTableProps } from './MetricsTable'; export interface ReferrersTableProps extends MetricsTableProps { allowFilter?: boolean; @@ -36,7 +36,7 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) { const renderLink = ({ x: referrer }) => { if (view === 'grouped') { - if (referrer === '_other') { + if (referrer === 'Other') { return `(${formatMessage(labels.other)})`; } else { return ( @@ -60,38 +60,13 @@ export function ReferrersTable({ allowFilter, ...props }: ReferrersTableProps) { ); }; - const getDomain = (x: string) => { - for (const { domain, match } of GROUPED_DOMAINS) { - if (Array.isArray(match) ? match.some(str => x.includes(str)) : x.includes(match)) { - return domain; - } - } - return '_other'; - }; - - const groupedFilter = (data: any[]) => { - const groups = { _other: 0 }; - - for (const { x, y } of data) { - const domain = getDomain(x); - if (!groups[domain]) { - groups[domain] = 0; - } - groups[domain] += +y; - } - - return Object.keys(groups) - .map((key: any) => ({ x: key, y: groups[key] })) - .sort(thenby.firstBy('y', -1)); - }; - return ( {allowFilter && } diff --git a/src/lib/constants.ts b/src/lib/constants.ts index 16bb71af..18bda293 100644 --- a/src/lib/constants.ts +++ b/src/lib/constants.ts @@ -26,11 +26,13 @@ export const REALTIME_RANGE = 30; export const REALTIME_INTERVAL = 10000; export const UNIT_TYPES = ['year', 'month', 'hour', 'day', 'minute']; + export const EVENT_COLUMNS = [ 'path', 'entry', 'exit', 'referrer', + 'grouped', 'title', 'query', 'event', @@ -59,6 +61,7 @@ export const FILTER_COLUMNS = { entry: 'url_path', exit: 'url_path', referrer: 'referrer_domain', + grouped: 'referrer_domain', hostname: 'hostname', title: 'page_title', query: 'url_query', @@ -386,6 +389,9 @@ export const GROUPED_DOMAINS = [ { name: 'Snapchat', domain: 'snapchat.com', match: 'snapchat.' }, { name: 'Pinterest', domain: 'pinterest.com', match: 'pinterest.' }, { name: 'ChatGPT', domain: 'chatgpt.com', match: 'chatgpt.' }, + { name: 'Yahoo', domain: 'yahoo.com', match: 'yahoo.' }, + { name: 'Yandex', domain: 'yandex.ru', match: 'yandex.' }, + { name: 'Baidu', domain: 'baidu.com', match: 'baidu.' }, ]; export const MAP_FILE = '/datamaps.world.json'; diff --git a/src/queries/sql/events/getWebsiteEvents.ts b/src/queries/sql/events/getWebsiteEvents.ts index 3b620018..579593d6 100644 --- a/src/queries/sql/events/getWebsiteEvents.ts +++ b/src/queries/sql/events/getWebsiteEvents.ts @@ -19,7 +19,7 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { search: `%${search}%`, }); - const searchQuery = filters.search + const searchQuery = search ? `and ((event_name ilike {{search}} and event_type = 2) or (url_path ilike {{search}} and event_type = 1))` : ''; @@ -59,12 +59,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { async function clickhouseQuery(websiteId: string, filters: QueryFilters) { const { pagedRawQuery, parseFilters } = clickhouse; + const { search } = filters; const { queryParams, dateQuery, cohortQuery, filterQuery } = parseFilters({ ...filters, websiteId, }); - const searchQuery = filters.search + const searchQuery = search ? `and ((positionCaseInsensitive(event_name, {search:String}) > 0 and event_type = 2) or (positionCaseInsensitive(url_path, {search:String}) > 0 and event_type = 1))` : ''; diff --git a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts index ebe31d7b..ce1f1df9 100644 --- a/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewExpandedMetrics.ts @@ -1,5 +1,5 @@ import clickhouse from '@/lib/clickhouse'; -import { EVENT_TYPE, FILTER_COLUMNS, SESSION_COLUMNS } from '@/lib/constants'; +import { EVENT_TYPE, FILTER_COLUMNS, GROUPED_DOMAINS, SESSION_COLUMNS } from '@/lib/constants'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; import { QueryFilters } from '@/lib/types'; @@ -99,7 +99,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { rawQuery, parseFilters } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -112,21 +112,24 @@ async function clickhouseQuery( if (column === 'referrer_domain') { excludeDomain = `and referrer_domain != hostname and referrer_domain != ''`; + if (type === 'grouped') { + column = toClickHouseGroupedReferrer(GROUPED_DOMAINS); + } } if (type === 'entry' || type === 'exit') { - const aggregrate = type === 'entry' ? 'min' : 'max'; + const aggregrate = type === 'entry' ? 'argMin' : 'argMax'; + column = `x.${column}`; entryExitQuery = ` JOIN (select visit_id, - ${aggregrate}(created_at) target_created_at + ${aggregrate}(url_path, created_at) url_path from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} group by visit_id) x - ON x.visit_id = website_event.visit_id - and x.target_created_at = website_event.created_at`; + ON x.visit_id = website_event.visit_id`; } return rawQuery( @@ -164,3 +167,19 @@ async function clickhouseQuery( { ...queryParams, ...parameters }, ); } + +export function toClickHouseGroupedReferrer( + domains: any[], + column: string = 'referrer_domain', +): string { + return [ + 'CASE', + ...domains.map(group => { + const matches = Array.isArray(group.match) ? group.match : [group.match]; + const formattedArray = matches.map(m => `'${m}'`).join(', '); + return ` WHEN multiSearchAny(${column}, [${formattedArray}]) != 0 THEN '${group.domain}'`; + }), + " ELSE 'Other'", + 'END', + ].join('\n'); +} diff --git a/src/queries/sql/pageviews/getPageviewMetrics.ts b/src/queries/sql/pageviews/getPageviewMetrics.ts index 4ebd4cd9..2403a8e8 100644 --- a/src/queries/sql/pageviews/getPageviewMetrics.ts +++ b/src/queries/sql/pageviews/getPageviewMetrics.ts @@ -30,7 +30,7 @@ async function relationalQuery( filters: QueryFilters, ): Promise { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { rawQuery, parseFilters } = prisma; const { filterQuery, joinSessionQuery, cohortQuery, queryParams } = parseFilters( { @@ -50,20 +50,21 @@ async function relationalQuery( } if (type === 'entry' || type === 'exit') { - const aggregrate = type === 'entry' ? 'min' : 'max'; + const order = type === 'entry' ? 'asc' : 'desc'; + column = `x.${column}`; entryExitQuery = ` join ( - select visit_id, - ${aggregrate}(created_at) target_created_at + select distinct on (visit_id) + visit_id, + url_path from website_event where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} - group by visit_id + order by visit_id, created_at ${order} ) x on x.visit_id = website_event.visit_id - and x.target_created_at = website_event.created_at `; } @@ -95,7 +96,7 @@ async function clickhouseQuery( filters: QueryFilters, ): Promise<{ x: string; y: number }[]> { const { type, limit = 500, offset = 0 } = parameters; - const column = FILTER_COLUMNS[type] || type; + let column = FILTER_COLUMNS[type] || type; const { rawQuery, parseFilters } = clickhouse; const { filterQuery, cohortQuery, queryParams } = parseFilters({ ...filters, @@ -114,18 +115,18 @@ async function clickhouseQuery( } if (type === 'entry' || type === 'exit') { - const aggregrate = type === 'entry' ? 'min' : 'max'; + const aggregrate = type === 'entry' ? 'argMin' : 'argMax'; + column = `x.${column}`; entryExitQuery = ` JOIN (select visit_id, - ${aggregrate}(created_at) target_created_at + ${aggregrate}(url_path, created_at) url_path from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime64} and {endDate:DateTime64} and event_type = {eventType:UInt32} group by visit_id) x - ON x.visit_id = website_event.visit_id - and x.target_created_at = website_event.created_at`; + ON x.visit_id = website_event.visit_id`; } sql = ` diff --git a/src/queries/sql/reports/getUTM.ts b/src/queries/sql/reports/getUTM.ts index f96c62d3..f690be1c 100644 --- a/src/queries/sql/reports/getUTM.ts +++ b/src/queries/sql/reports/getUTM.ts @@ -1,16 +1,15 @@ import clickhouse from '@/lib/clickhouse'; import { CLICKHOUSE, PRISMA, runQuery } from '@/lib/db'; import prisma from '@/lib/prisma'; +import { QueryFilters } from '@/lib/types'; + +export interface UTMParameters { + startDate: Date; + endDate: Date; +} export async function getUTM( - ...args: [ - websiteId: string, - filters: { - startDate: Date; - endDate: Date; - timezone?: string; - }, - ] + ...args: [websiteId: string, parameters: UTMParameters, filters: QueryFilters] ) { return runQuery({ [PRISMA]: () => relationalQuery(...args), @@ -20,14 +19,18 @@ export async function getUTM( async function relationalQuery( websiteId: string, - filters: { - startDate: Date; - endDate: Date; - timezone?: string; - }, + parameters: UTMParameters, + filters: QueryFilters, ) { - const { startDate, endDate } = filters; - const { rawQuery } = prisma; + const { startDate, endDate } = parameters; + const { parseFilters, rawQuery } = prisma; + + const { filterQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + startDate, + endDate, + }); return rawQuery( ` @@ -37,26 +40,26 @@ async function relationalQuery( and created_at between {{startDate}} and {{endDate}} and coalesce(url_query, '') != '' and event_type = 1 + ${filterQuery} group by 1 `, - { - websiteId, - startDate, - endDate, - }, + queryParams, ); } async function clickhouseQuery( websiteId: string, - filters: { - startDate: Date; - endDate: Date; - timezone?: string; - }, + parameters: UTMParameters, + filters: QueryFilters, ) { - const { startDate, endDate } = filters; - const { rawQuery } = clickhouse; + const { startDate, endDate } = parameters; + const { parseFilters, rawQuery } = clickhouse; + const { filterQuery, queryParams } = parseFilters({ + ...filters, + websiteId, + startDate, + endDate, + }); return rawQuery( ` @@ -66,12 +69,9 @@ async function clickhouseQuery( and created_at between {startDate:DateTime64} and {endDate:DateTime64} and url_query != '' and event_type = 1 + ${filterQuery} group by 1 `, - { - websiteId, - startDate, - endDate, - }, + queryParams, ); } diff --git a/src/queries/sql/sessions/getWebsiteSessions.ts b/src/queries/sql/sessions/getWebsiteSessions.ts index fdc1bf9c..c0942b20 100644 --- a/src/queries/sql/sessions/getWebsiteSessions.ts +++ b/src/queries/sql/sessions/getWebsiteSessions.ts @@ -20,7 +20,13 @@ async function relationalQuery(websiteId: string, filters: QueryFilters) { search: search ? `%${search}%` : undefined, }); - const searchQuery = search ? `and session.distinct_id ilike {{search}}` : ''; + const searchQuery = search + ? `and (distinct_id ilike {{search}} + or city ilike {{search}} + or browser ilike {{search}} + or os ilike {{search}} + or device ilike {{search}})` + : ''; return pagedRawQuery( ` @@ -74,7 +80,13 @@ async function clickhouseQuery(websiteId: string, filters: QueryFilters) { websiteId, }); - const searchQuery = search ? `and positionCaseInsensitive(distinct_id, {search:String}) > 0` : ''; + const searchQuery = search + ? `and ((positionCaseInsensitive(distinct_id, {search:String}) > 0) + or (positionCaseInsensitive(city, {search:String}) > 0) + or (positionCaseInsensitive(browser, {search:String}) > 0) + or (positionCaseInsensitive(os, {search:String}) > 0) + or (positionCaseInsensitive(device, {search:String}) > 0))` + : ''; let sql = '';