diff --git a/.github/workflows/cd-manual.yml b/.github/workflows/cd-manual.yml index 1afc6e93..ac701fcc 100644 --- a/.github/workflows/cd-manual.yml +++ b/.github/workflows/cd-manual.yml @@ -21,7 +21,7 @@ jobs: - uses: actions/checkout@v3 - uses: mr-smithers-excellent/docker-build-push@v6 - name: Build & push Docker image for ${{ matrix.db-type }} + name: Build & push Docker image to ghcr.io for ${{ matrix.db-type }} with: image: umami tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest @@ -31,3 +31,13 @@ jobs: platform: linux/amd64,linux/arm64 username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + + - uses: mr-smithers-excellent/docker-build-push@v6 + name: Build & push Docker image to docker.io for ${{ matrix.db-type }} + with: + image: umamisoftware/umami + tags: ${{ matrix.db-type }}-${{ inputs.version }}, ${{ matrix.db-type }}-latest + buildArgs: DATABASE_TYPE=${{ matrix.db-type }} + registry: docker.io + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 6fda05a6..0660bcba 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -19,7 +19,7 @@ jobs: run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - uses: mr-smithers-excellent/docker-build-push@v6 - name: Build & push Docker image for ${{ matrix.db-type }} + name: Build & push Docker image to ghcr.io for ${{ matrix.db-type }} with: image: umami tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest @@ -29,3 +29,13 @@ jobs: platform: linux/amd64,linux/arm64 username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + + - uses: mr-smithers-excellent/docker-build-push@v6 + name: Build & push Docker image to docker.io for ${{ matrix.db-type }} + with: + image: umamisoftware/umami + tags: ${{ matrix.db-type }}-${{ env.RELEASE_VERSION }}, ${{ matrix.db-type }}-latest + buildArgs: DATABASE_TYPE=${{ matrix.db-type }} + registry: docker.io + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} \ No newline at end of file diff --git a/components/layout/AppLayout.js b/components/layout/AppLayout.js index 989128f9..7ab74351 100644 --- a/components/layout/AppLayout.js +++ b/components/layout/AppLayout.js @@ -2,9 +2,7 @@ import { Container } from 'react-basics'; import Head from 'next/head'; import NavBar from 'components/layout/NavBar'; import UpdateNotice from 'components/common/UpdateNotice'; -import useRequireLogin from 'hooks/useRequireLogin'; -import useConfig from 'hooks/useConfig'; -import { CURRENT_VERSION } from 'lib/constants'; +import { useRequireLogin, useConfig } from 'hooks'; import styles from './AppLayout.module.css'; export function AppLayout({ title, children }) { @@ -16,7 +14,7 @@ export function AppLayout({ title, children }) { } return ( -
+
{title ? `${title} | umami` : 'umami'} diff --git a/components/pages/reports/FieldSelectForm.js b/components/pages/reports/FieldSelectForm.js index 0e41ea1f..69f399bb 100644 --- a/components/pages/reports/FieldSelectForm.js +++ b/components/pages/reports/FieldSelectForm.js @@ -13,7 +13,7 @@ export default function FieldSelectForm({ fields, onSelect }) { return (
{label || name}
-
{type}
+ {type &&
{type}
} ); })} diff --git a/components/pages/reports/ReportTemplates.js b/components/pages/reports/ReportTemplates.js index 29c193a8..1de7de9c 100644 --- a/components/pages/reports/ReportTemplates.js +++ b/components/pages/reports/ReportTemplates.js @@ -33,14 +33,12 @@ export function ReportTemplates() { const { formatMessage, labels } = useMessages(); const reports = [ - /* { title: formatMessage(labels.insights), description: 'Dive deeper into your data by using segments and filters.', url: '/reports/insights', icon: , }, - */ { title: formatMessage(labels.funnel), description: 'Understand the conversion and drop-off rate of users.', diff --git a/components/pages/reports/funnel/FunnelParameters.js b/components/pages/reports/funnel/FunnelParameters.js index ae498176..03898db3 100644 --- a/components/pages/reports/funnel/FunnelParameters.js +++ b/components/pages/reports/funnel/FunnelParameters.js @@ -16,6 +16,7 @@ import UrlAddForm from './UrlAddForm'; import { ReportContext } from 'components/pages/reports/Report'; import BaseParameters from '../BaseParameters'; import ParameterList from '../ParameterList'; +import PopupForm from '../PopupForm'; export function FunnelParameters() { const { report, runReport, updateReport, isRunning } = useContext(ReportContext); @@ -53,7 +54,11 @@ export function FunnelParameters() { {(close, element) => { - return ; + return ( + + + + ); }} diff --git a/components/pages/reports/funnel/UrlAddForm.js b/components/pages/reports/funnel/UrlAddForm.js index 0fb78b3d..ce202116 100644 --- a/components/pages/reports/funnel/UrlAddForm.js +++ b/components/pages/reports/funnel/UrlAddForm.js @@ -2,16 +2,14 @@ import { useState } from 'react'; import { useMessages } from 'hooks'; import { Button, Form, FormRow, TextField, Flexbox } from 'react-basics'; import styles from './UrlAddForm.module.css'; -import PopupForm from '../PopupForm'; -export function UrlAddForm({ defaultValue = '', element, onAdd, onClose }) { +export function UrlAddForm({ defaultValue = '', onAdd }) { const [url, setUrl] = useState(defaultValue); const { formatMessage, labels } = useMessages(); const handleSave = () => { onAdd(url); setUrl(''); - onClose(); }; const handleChange = e => { @@ -26,25 +24,23 @@ export function UrlAddForm({ defaultValue = '', element, onAdd, onClose }) { }; return ( - -
- - - - - - -
- +
+ + + + + + +
); } diff --git a/components/pages/reports/insights/InsightsParameters.js b/components/pages/reports/insights/InsightsParameters.js index b87a566d..692c5ead 100644 --- a/components/pages/reports/insights/InsightsParameters.js +++ b/components/pages/reports/insights/InsightsParameters.js @@ -2,12 +2,28 @@ import { useContext, useRef } from 'react'; import { useMessages } from 'hooks'; import { Form, FormRow, FormButtons, SubmitButton, PopupTrigger, Icon, Popup } from 'react-basics'; import { ReportContext } from 'components/pages/reports/Report'; -import { REPORT_PARAMETERS, WEBSITE_EVENT_FIELDS } from 'lib/constants'; +import { REPORT_PARAMETERS } from 'lib/constants'; import Icons from 'components/icons'; import BaseParameters from '../BaseParameters'; -import FieldAddForm from '../FieldAddForm'; import ParameterList from '../ParameterList'; import styles from './InsightsParameters.module.css'; +import FieldSelectForm from '../FieldSelectForm'; +import PopupForm from '../PopupForm'; +import FieldFilterForm from '../FieldFilterForm'; + +const fieldOptions = [ + { name: 'url', type: 'string' }, + { name: 'title', type: 'string' }, + { name: 'referrer', type: 'string' }, + { name: 'query', type: 'string' }, + { name: 'browser', type: 'string' }, + { name: 'os', type: 'string' }, + { name: 'device', type: 'string' }, + { name: 'country', type: 'string' }, + { name: 'region', type: 'string' }, + { name: 'city', type: 'string' }, + { name: 'language', type: 'string' }, +]; export function InsightsParameters() { const { report, runReport, updateReport, isRunning } = useContext(ReportContext); @@ -16,7 +32,6 @@ export function InsightsParameters() { const { parameters } = report || {}; const { websiteId, dateRange, fields, filters, groups } = parameters || {}; const queryEnabled = websiteId && dateRange && fields?.length; - const fieldOptions = Object.keys(WEBSITE_EVENT_FIELDS).map(key => WEBSITE_EVENT_FIELDS[key]); const parameterGroups = [ { label: formatMessage(labels.fields), group: REPORT_PARAMETERS.fields }, @@ -57,13 +72,14 @@ export function InsightsParameters() { {(close, element) => { return ( - + + {group === REPORT_PARAMETERS.fields && ( + + )} + {group === REPORT_PARAMETERS.filters && ( + + )} + ); }} diff --git a/components/pages/reports/insights/InsightsTable.js b/components/pages/reports/insights/InsightsTable.js index a767468e..d751445b 100644 --- a/components/pages/reports/insights/InsightsTable.js +++ b/components/pages/reports/insights/InsightsTable.js @@ -6,11 +6,13 @@ import { ReportContext } from '../Report'; export function InsightsTable() { const { report } = useContext(ReportContext); const { formatMessage, labels } = useMessages(); + const { fields = [] } = report?.parameters || {}; return ( - - + {fields.map(({ name }) => { + return ; + })} ); diff --git a/lib/clickhouse.ts b/lib/clickhouse.ts index b3dc2c48..19d09405 100644 --- a/lib/clickhouse.ts +++ b/lib/clickhouse.ts @@ -61,14 +61,17 @@ function getDateFormat(date) { return `'${dateFormat(date, 'UTC:yyyy-mm-dd HH:MM:ss')}'`; } -function getFilterQuery(filters = {}, params = {}) { +function getFilterQuery(filters = {}) { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; if (filter !== undefined) { const column = FILTER_COLUMNS[key] || key; arr.push(`and ${column} = {${key}:String}`); - params[key] = decodeURIComponent(filter); + } + + if (key === 'referrer') { + arr.push('and referrer_domain != {domain:String}'); } return arr; @@ -77,9 +80,9 @@ function getFilterQuery(filters = {}, params = {}) { return query.join('\n'); } -function parseFilters(filters: WebsiteMetricFilter = {}, params: any = {}) { +function parseFilters(filters: WebsiteMetricFilter = {}) { return { - filterQuery: getFilterQuery(filters, params), + filterQuery: getFilterQuery(filters), }; } diff --git a/lib/constants.ts b/lib/constants.ts index c275ed8d..2b3da875 100644 --- a/lib/constants.ts +++ b/lib/constants.ts @@ -43,11 +43,6 @@ export const SESSION_COLUMNS = [ 'city', ]; -export const COLLECTION_TYPE = { - event: 'event', - identify: 'identify', -}; - export const FILTER_COLUMNS = { url: 'url_path', referrer: 'referrer_domain', @@ -57,6 +52,11 @@ export const FILTER_COLUMNS = { region: 'subdivision1', }; +export const COLLECTION_TYPE = { + event: 'event', + identify: 'identify', +}; + export const EVENT_TYPE = { pageView: 1, customEvent: 2, @@ -120,37 +120,6 @@ export const ROLE_PERMISSIONS = { [ROLES.teamMember]: [], } as const; -export const WEBSITE_EVENT_FIELDS = { - eventId: { name: 'event_id', type: 'uuid', label: 'Event ID' }, - websiteId: { name: 'website_id', type: 'uuid', label: 'Website ID' }, - sessionId: { name: 'session_id', type: 'uuid', label: 'Session ID' }, - createdAt: { name: 'created_at', type: 'date', label: 'Created date' }, - urlPath: { name: 'url_path', type: 'string', label: 'URL path' }, - urlQuery: { name: 'url_query', type: 'string', label: 'URL query' }, - referrerPath: { name: 'referrer_path', type: 'string', label: 'Referrer path' }, - referrerQuery: { name: 'referrer_query', type: 'string', label: 'Referrer query' }, - referrerDomain: { name: 'referrer_domain', type: 'string', label: 'Referrer domain' }, - pageTitle: { name: 'page_title', type: 'string', label: 'Page title' }, - eventType: { name: 'event_type', type: 'string', label: 'Event type' }, - eventName: { name: 'event_name', type: 'string', label: 'Event name' }, -}; - -export const SESSION_FIELDS = { - sessionId: { name: 'session_id', type: 'uuid' }, - websiteId: { name: 'website_id', type: 'uuid' }, - hostname: { name: 'hostname', type: 'string' }, - browser: { name: 'browser', type: 'string' }, - os: { name: 'os', type: 'string' }, - device: { name: 'device', type: 'string' }, - screen: { name: 'screen', type: 'string' }, - language: { name: 'language', type: 'string' }, - country: { name: 'country', type: 'string' }, - subdivision1: { name: 'subdivision1', type: 'string' }, - subdivision2: { name: 'subdivision2', type: 'string' }, - city: { name: 'city', type: 'string' }, - createdAt: { name: 'created_at', type: 'date' }, -}; - export const THEME_COLORS = { light: { primary: '#2680eb', diff --git a/lib/prisma.ts b/lib/prisma.ts index 08309e31..a6f1ff88 100644 --- a/lib/prisma.ts +++ b/lib/prisma.ts @@ -1,7 +1,7 @@ import prisma from '@umami/prisma-client'; import moment from 'moment-timezone'; import { MYSQL, POSTGRESQL, getDatabaseType } from 'lib/db'; -import { FILTER_COLUMNS } from './constants'; +import { FILTER_COLUMNS, SESSION_COLUMNS } from './constants'; const MYSQL_DATE_FORMATS = { minute: '%Y-%m-%d %H:%i:00', @@ -64,14 +64,19 @@ function getTimestampIntervalQuery(field: string): string { } } -function getFilterQuery(filters = {}, params = []): string { +function getFilterQuery(filters = {}): string { const query = Object.keys(filters).reduce((arr, key) => { const filter = filters[key]; if (filter !== undefined) { const column = FILTER_COLUMNS[key] || key; arr.push(`and ${column}={{${key}}}`); - params.push(decodeURIComponent(filter)); + } + + if (key === 'referrer') { + arr.push( + 'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)', + ); } return arr; @@ -80,19 +85,12 @@ function getFilterQuery(filters = {}, params = []): string { return query.join('\n'); } -function parseFilters( - filters: { [key: string]: any } = {}, - params = [], - sessionKey = 'session_id', -) { - const { os, browser, device, country, region, city } = filters; - +function parseFilters(filters: { [key: string]: any } = {}) { return { - joinSession: - os || browser || device || country || region || city - ? `inner join session on website_event.${sessionKey} = session.${sessionKey}` - : '', - filterQuery: getFilterQuery(filters, params), + joinSession: Object.keys(filters).find(key => SESSION_COLUMNS[key]) + ? `inner join session on website_event.session_id = session.session_id` + : '', + filterQuery: getFilterQuery(filters), }; } diff --git a/pages/api/auth/login.ts b/pages/api/auth/login.ts index af206938..b9a2be00 100644 --- a/pages/api/auth/login.ts +++ b/pages/api/auth/login.ts @@ -7,6 +7,7 @@ import { checkPassword, createSecureToken, methodNotAllowed, + forbidden, } from 'next-basics'; import redis from '@umami/redis-client'; import { getUserByUsername } from 'queries'; @@ -30,6 +31,10 @@ export default async ( req: NextApiRequestQueryBody, res: NextApiResponse, ) => { + if (process.env.DISABLE_LOGIN) { + return forbidden(res); + } + if (req.method === 'POST') { const { username, password } = req.body; diff --git a/pages/api/reports/insights.ts b/pages/api/reports/insights.ts index dba11953..a40c2124 100644 --- a/pages/api/reports/insights.ts +++ b/pages/api/reports/insights.ts @@ -11,7 +11,7 @@ export interface InsightsRequestBody { startDate: string; endDate: string; }; - fields: string[]; + fields: { name: string; type: string; value: string }[]; filters: string[]; groups: string[]; } diff --git a/pages/api/websites/[id]/metrics.ts b/pages/api/websites/[id]/metrics.ts index 37a04691..15389e3e 100644 --- a/pages/api/websites/[id]/metrics.ts +++ b/pages/api/websites/[id]/metrics.ts @@ -46,6 +46,7 @@ export default async ( country, region, city, + language, } = req.query; if (req.method === 'GET') { @@ -55,20 +56,27 @@ export default async ( const { startDate, endDate } = await parseDateRangeQuery(req); + const filters = { + url, + referrer, + title, + query, + event, + os, + browser, + device, + country, + region, + city, + language, + }; + + filters[type] = undefined; + + const column = FILTER_COLUMNS[type] || type; + if (SESSION_COLUMNS.includes(type)) { - const column = FILTER_COLUMNS[type] || type; - const filters = { - os, - browser, - device, - country, - region, - city, - }; - - filters[type] = undefined; - - let data = await getSessionMetrics(websiteId, { + const data = await getSessionMetrics(websiteId, { startDate, endDate, column, @@ -88,30 +96,13 @@ export default async ( } } - data = Object.values(combined); + return ok(res, Object.values(combined)); } return ok(res, data); } if (EVENT_COLUMNS.includes(type)) { - const column = FILTER_COLUMNS[type] || type; - const filters = { - url, - referrer, - title, - query, - event, - os, - browser, - device, - country, - region, - city, - }; - - filters[type] = undefined; - const data = await getPageviewMetrics(websiteId, { startDate, endDate, diff --git a/pages/api/websites/[id]/pageviews.ts b/pages/api/websites/[id]/pageviews.ts index 453c6733..810854a7 100644 --- a/pages/api/websites/[id]/pageviews.ts +++ b/pages/api/websites/[id]/pageviews.ts @@ -6,6 +6,7 @@ import { canViewWebsite } from 'lib/auth'; import { useAuth, useCors } from 'lib/middleware'; import { getPageviewStats } from 'queries'; import { parseDateRangeQuery } from 'lib/query'; +import { getSessionStats } from '../../../../queries/analytics/sessions/getSessionStats'; export interface WebsitePageviewRequestQuery { id: string; @@ -62,7 +63,6 @@ export default async ( endDate, timezone, unit, - count: '*', filters: { url, referrer, @@ -75,14 +75,14 @@ export default async ( city, }, }), - getPageviewStats(websiteId, { + getSessionStats(websiteId, { startDate, endDate, timezone, unit, - count: 'distinct website_event.', filters: { url, + referrer, title, os, browser, diff --git a/queries/analytics/events/getEventMetrics.ts b/queries/analytics/events/getEventMetrics.ts index e9754036..03b252b7 100644 --- a/queries/analytics/events/getEventMetrics.ts +++ b/queries/analytics/events/getEventMetrics.ts @@ -47,11 +47,12 @@ async function relationalQuery(websiteId: string, criteria: GetEventMetricsCrite order by 2 `, { + ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate, eventType: EVENT_TYPE.customEvent, - ...filters, + domain: website.domain, }, ); } @@ -82,6 +83,7 @@ async function clickhouseQuery(websiteId: string, criteria: GetEventMetricsCrite startDate: maxDate(startDate, website.resetAt), endDate, eventType: EVENT_TYPE.customEvent, + domain: website.domain, }, ); } diff --git a/queries/analytics/pageviews/getPageviewMetrics.ts b/queries/analytics/pageviews/getPageviewMetrics.ts index 1032540b..8e4460e6 100644 --- a/queries/analytics/pageviews/getPageviewMetrics.ts +++ b/queries/analytics/pageviews/getPageviewMetrics.ts @@ -34,22 +34,6 @@ async function relationalQuery( const { startDate, endDate, filters = {}, column } = criteria; const { rawQuery, parseFilters } = prisma; const website = await loadWebsite(websiteId); - const params: any = { - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, - ...filters, - }; - - let excludeDomain = ''; - - if (column === 'referrer_domain') { - excludeDomain = - 'and (website_event.referrer_domain != {{domain}} or website_event.referrer_domain is null)'; - - params.domain = website.domain; - } const { filterQuery, joinSession } = parseFilters(filters); @@ -61,13 +45,19 @@ async function relationalQuery( where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} and event_type = {{eventType}} - ${excludeDomain} ${filterQuery} group by 1 order by 2 desc limit 100 `, - params, + { + ...filters, + websiteId, + startDate: maxDate(startDate, website.resetAt), + endDate, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + domain: website.domain, + }, ); } @@ -83,22 +73,8 @@ async function clickhouseQuery( const { startDate, endDate, filters = {}, column } = criteria; const { rawQuery, parseFilters } = clickhouse; const website = await loadWebsite(websiteId); - const params = { - websiteId, - startDate: maxDate(startDate, website.resetAt), - endDate, - eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, - domain: undefined, - }; - let excludeDomain = ''; - - if (column === 'referrer_domain') { - excludeDomain = 'and referrer_domain != {domain:String}'; - params.domain = website.domain; - } - - const { filterQuery } = parseFilters(filters, params); + const { filterQuery } = parseFilters(filters); return rawQuery( ` @@ -107,12 +83,18 @@ async function clickhouseQuery( where website_id = {websiteId:UUID} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} - ${excludeDomain} ${filterQuery} group by x order by y desc limit 100 `, - params, + { + ...filters, + websiteId, + startDate: maxDate(startDate, website.resetAt), + endDate, + eventType: column === 'event_name' ? EVENT_TYPE.customEvent : EVENT_TYPE.pageView, + domain: website.domain, + }, ); } diff --git a/queries/analytics/pageviews/getPageviewStats.ts b/queries/analytics/pageviews/getPageviewStats.ts index f6d4158c..cdbd6442 100644 --- a/queries/analytics/pageviews/getPageviewStats.ts +++ b/queries/analytics/pageviews/getPageviewStats.ts @@ -10,9 +10,19 @@ export interface PageviewStatsCriteria { endDate: Date; timezone?: string; unit?: string; - count?: string; - filters: object; - sessionKey?: string; + filters: { + url?: string; + referrer?: string; + title?: string; + browser?: string; + os?: string; + device?: string; + screen?: string; + language?: string; + country?: string; + region?: string; + city?: string; + }; } export async function getPageviewStats( @@ -25,15 +35,7 @@ export async function getPageviewStats( } async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteria) { - const { - startDate, - endDate, - timezone = 'utc', - unit = 'day', - count = '*', - filters = {}, - sessionKey = 'session_id', - } = criteria; + const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria; const { getDateQuery, parseFilters, rawQuery } = prisma; const website = await loadWebsite(websiteId); const { filterQuery, joinSession } = parseFilters(filters); @@ -42,7 +44,7 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri ` select ${getDateQuery('website_event.created_at', unit, timezone)} x, - count(${count !== '*' ? `${count}${sessionKey}` : count}) y + count(*) y from website_event ${joinSession} where website_event.website_id = {{websiteId::uuid}} @@ -52,24 +54,18 @@ async function relationalQuery(websiteId: string, criteria: PageviewStatsCriteri group by 1 `, { + ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate, eventType: EVENT_TYPE.pageView, - ...filters, + domain: website.domain, }, ); } async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteria) { - const { - startDate, - endDate, - timezone = 'UTC', - unit = 'day', - count = '*', - filters = {}, - } = criteria; + const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria; const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; const website = await loadWebsite(websiteId); const { filterQuery } = parseFilters(filters); @@ -82,7 +78,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri from ( select ${getDateQuery('created_at', unit, timezone)} as t, - count(${count !== '*' ? 'distinct session_id' : count}) as y + count(*) as y from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime} and {endDate:DateTime} @@ -98,6 +94,7 @@ async function clickhouseQuery(websiteId: string, criteria: PageviewStatsCriteri startDate: maxDate(startDate, website.resetAt), endDate, eventType: EVENT_TYPE.pageView, + domain: website.domain, }, ); } diff --git a/queries/analytics/reports/getInsights.ts b/queries/analytics/reports/getInsights.ts index 1d8970ed..ff139931 100644 --- a/queries/analytics/reports/getInsights.ts +++ b/queries/analytics/reports/getInsights.ts @@ -1,11 +1,14 @@ import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; +import { maxDate } from 'lib/date'; +import { EVENT_TYPE } from 'lib/constants'; +import { loadWebsite } from 'lib/load'; export interface GetInsightsCriteria { startDate: Date; endDate: Date; - fields: string[]; + fields: { name: string; type: string; value: string }[]; filters: string[]; groups: string[]; } @@ -26,7 +29,33 @@ async function relationalQuery( y: number; }[] > { - return null; + const { startDate, endDate, filters = [] } = criteria; + const { parseFilters, rawQuery } = prisma; + const website = await loadWebsite(websiteId); + const params = {}; + const { filterQuery, joinSession } = parseFilters(params); + + return rawQuery( + ` + select + url_path, + count(*) y + from website_event + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and website_event.event_type = {{eventType}} + ${filterQuery} + group by 1 + `, + { + ...filters, + websiteId, + startDate: maxDate(startDate, website.resetAt), + endDate, + eventType: EVENT_TYPE.pageView, + }, + ); } async function clickhouseQuery( @@ -38,5 +67,54 @@ async function clickhouseQuery( y: number; }[] > { - return null; + const { startDate, endDate, fields = [], filters = [], groups = [] } = criteria; + const { parseFilters, rawQuery } = clickhouse; + const website = await loadWebsite(websiteId); + const params = {}; + const { filterQuery } = parseFilters(params); + + const fieldsQuery = parseFields(fields); + + return rawQuery( + ` + select + ${fieldsQuery} + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime} and {endDate:DateTime} + and event_type = {eventType:UInt32} + ${filterQuery} + group by ${fields.map(({ name }) => name).join(',')} + order by total desc + limit 500 + `, + { + ...filters, + websiteId, + startDate: maxDate(startDate, website.resetAt), + endDate, + eventType: EVENT_TYPE.pageView, + }, + ); +} + +function parseFields(fields) { + let count = false; + let distinct = false; + + const query = fields.reduce((arr, field) => { + const { name, value } = field; + + if (!count && value === 'total') { + count = true; + arr = arr.concat(`count(*) as views`); + } else if (!distinct && value === 'unique') { + distinct = true; + //arr = arr.concat(`count(distinct ${name})`); + } + + return arr.concat(name); + }, []); + + return query.join(',\n'); } diff --git a/queries/analytics/sessions/getSessionMetrics.ts b/queries/analytics/sessions/getSessionMetrics.ts index aec2d8f1..a9b49ec8 100644 --- a/queries/analytics/sessions/getSessionMetrics.ts +++ b/queries/analytics/sessions/getSessionMetrics.ts @@ -1,7 +1,7 @@ import prisma from 'lib/prisma'; import clickhouse from 'lib/clickhouse'; import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db'; -import { DEFAULT_RESET_DATE, EVENT_TYPE } from 'lib/constants'; +import { EVENT_TYPE } from 'lib/constants'; import { loadWebsite } from 'lib/load'; import { maxDate } from 'lib/date'; @@ -28,17 +28,12 @@ async function relationalQuery( return rawQuery( `select ${column} x, count(*) y - from session as x - where x.session_id in ( - select website_event.session_id from website_event - join website - on website_event.website_id = website.website_id - ${joinSession} - where website.website_id = {{websiteId::uuid}} + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} and website_event.created_at between {{startDate}} and {{endDate}} ${filterQuery} - ) + ) as t group by 1 order by 2 desc limit 100`, @@ -64,7 +59,7 @@ async function clickhouseQuery( ` select ${column} x, count(distinct session_id) y - from website_event as x + from website_event where website_id = {websiteId:UUID} and created_at between {startDate:DateTime} and {endDate:DateTime} and event_type = {eventType:UInt32} diff --git a/queries/analytics/sessions/getSessionStats.ts b/queries/analytics/sessions/getSessionStats.ts new file mode 100644 index 00000000..7633f242 --- /dev/null +++ b/queries/analytics/sessions/getSessionStats.ts @@ -0,0 +1,100 @@ +import clickhouse from 'lib/clickhouse'; +import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db'; +import prisma from 'lib/prisma'; +import { EVENT_TYPE } from 'lib/constants'; +import { loadWebsite } from 'lib/load'; +import { maxDate } from 'lib/date'; + +export interface SessionStatsCriteria { + startDate: Date; + endDate: Date; + timezone?: string; + unit?: string; + filters: { + url?: string; + referrer?: string; + title?: string; + browser?: string; + os?: string; + device?: string; + screen?: string; + language?: string; + country?: string; + region?: string; + city?: string; + }; +} + +export async function getSessionStats( + ...args: [websiteId: string, criteria: SessionStatsCriteria] +) { + return runQuery({ + [PRISMA]: () => relationalQuery(...args), + [CLICKHOUSE]: () => clickhouseQuery(...args), + }); +} + +async function relationalQuery(websiteId: string, criteria: SessionStatsCriteria) { + const { startDate, endDate, timezone = 'utc', unit = 'day', filters = {} } = criteria; + const { getDateQuery, parseFilters, rawQuery } = prisma; + const website = await loadWebsite(websiteId); + const { filterQuery, joinSession } = parseFilters(filters); + + return rawQuery( + ` + select + ${getDateQuery('website_event.created_at', unit, timezone)} x, + count(distinct website_event.session_id) y + from website_event + ${joinSession} + where website_event.website_id = {{websiteId::uuid}} + and website_event.created_at between {{startDate}} and {{endDate}} + and event_type = {{eventType}} + ${filterQuery} + group by 1 + `, + { + ...filters, + websiteId, + startDate: maxDate(startDate, website.resetAt), + endDate, + eventType: EVENT_TYPE.pageView, + domain: website.domain, + }, + ); +} + +async function clickhouseQuery(websiteId: string, criteria: SessionStatsCriteria) { + const { startDate, endDate, timezone = 'UTC', unit = 'day', filters = {} } = criteria; + const { parseFilters, rawQuery, getDateStringQuery, getDateQuery } = clickhouse; + const website = await loadWebsite(websiteId); + const { filterQuery } = parseFilters(filters); + + return rawQuery( + ` + select + ${getDateStringQuery('g.t', unit)} as x, + g.y as y + from ( + select + ${getDateQuery('created_at', unit, timezone)} as t, + count(distinct session_id) as y + from website_event + where website_id = {websiteId:UUID} + and created_at between {startDate:DateTime} and {endDate:DateTime} + and event_type = {eventType:UInt32} + ${filterQuery} + group by t + ) as g + order by t + `, + { + ...filters, + websiteId, + startDate: maxDate(startDate, website.resetAt), + endDate, + eventType: EVENT_TYPE.pageView, + domain: website.domain, + }, + ); +} diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/stats/getWebsiteStats.ts index 4d3730ee..e048fc8f 100644 --- a/queries/analytics/stats/getWebsiteStats.ts +++ b/queries/analytics/stats/getWebsiteStats.ts @@ -51,11 +51,12 @@ async function relationalQuery( ) as t `, { + ...filters, websiteId, startDate: maxDate(startDate, website.resetAt), endDate, eventType: EVENT_TYPE.pageView, - ...filters, + domain: website.domain, }, ); } @@ -97,6 +98,7 @@ async function clickhouseQuery( startDate: maxDate(startDate, website.resetAt), endDate, eventType: EVENT_TYPE.pageView, + domain: website.domain, }, ); }