From 618c643a0a16f69d8b7c1d6b856a69619b9a3bb3 Mon Sep 17 00:00:00 2001
From: Mike Cao <mike@mikecao.com>
Date: Tue, 8 Aug 2023 15:29:59 -0700
Subject: [PATCH] Insights report filtering.

---
 components/pages/reports/FieldFilterForm.js   | 45 +++++++------------
 components/pages/reports/FilterSelectForm.js  | 25 ++++++++++-
 .../reports/insights/InsightsParameters.js    |  6 ++-
 hooks/useFilters.js                           |  4 +-
 pages/api/websites/[id]/values.ts             | 43 ++++++++++++++++++
 .../{stats => }/getActiveVisitors.ts          |  6 +--
 .../analytics/{stats => }/getRealtimeData.ts  |  2 +-
 queries/analytics/getValues.ts                | 38 ++++++++++++++++
 .../{stats => }/getWebsiteDateRange.ts        |  0
 .../analytics/{stats => }/getWebsiteStats.ts  |  0
 queries/analytics/reports/getInsights.ts      | 26 ++++++-----
 queries/index.js                              |  9 ++--
 12 files changed, 152 insertions(+), 52 deletions(-)
 create mode 100644 pages/api/websites/[id]/values.ts
 rename queries/analytics/{stats => }/getActiveVisitors.ts (84%)
 rename queries/analytics/{stats => }/getRealtimeData.ts (93%)
 create mode 100644 queries/analytics/getValues.ts
 rename queries/analytics/{stats => }/getWebsiteDateRange.ts (100%)
 rename queries/analytics/{stats => }/getWebsiteStats.ts (100%)

diff --git a/components/pages/reports/FieldFilterForm.js b/components/pages/reports/FieldFilterForm.js
index 021ea97e..a2b68968 100644
--- a/components/pages/reports/FieldFilterForm.js
+++ b/components/pages/reports/FieldFilterForm.js
@@ -1,48 +1,37 @@
 import { useState } from 'react';
-import { Form, FormRow, Menu, Item, Flexbox, Dropdown, TextField, Button } from 'react-basics';
+import { Form, FormRow, Item, Flexbox, Dropdown, Button } from 'react-basics';
 import { useFilters } from 'hooks';
 import styles from './FieldFilterForm.module.css';
 
-export default function FieldFilterForm({ name, type, onSelect }) {
-  const [filter, setFilter] = useState('');
-  const [value, setValue] = useState('');
-  const { filters, types } = useFilters();
-  const items = types[type];
+export default function FieldFilterForm({ label, type, values, onSelect }) {
+  const [filter, setFilter] = useState('eq');
+  const [value, setValue] = useState();
+  const filters = useFilters(type);
 
-  const renderValue = value => {
-    return filters[value];
+  const renderFilterValue = value => {
+    return filters.find(f => f.value === value)?.label;
   };
 
-  if (type === 'boolean') {
-    return (
-      <Form>
-        <FormRow label={name}>
-          <Menu onSelect={value => onSelect({ name, type, value: ['eq', value] })}>
-            {items.map(value => {
-              return <Item key={value}>{filters[value]}</Item>;
-            })}
-          </Menu>
-        </FormRow>
-      </Form>
-    );
-  }
-
   return (
     <Form>
-      <FormRow label={name} className={styles.filter}>
+      <FormRow label={label} className={styles.filter}>
         <Flexbox gap={10}>
           <Dropdown
             className={styles.dropdown}
-            items={items}
+            items={filters}
             value={filter}
-            renderValue={renderValue}
+            renderValue={renderFilterValue}
             onChange={setFilter}
           >
-            {value => {
-              return <Item key={value}>{filters[value]}</Item>;
+            {({ value, label }) => {
+              return <Item key={value}>{label}</Item>;
+            }}
+          </Dropdown>
+          <Dropdown className={styles.values} items={values} value={value} onChange={setValue}>
+            {value => {
+              return <Item key={value}>{value}</Item>;
             }}
           </Dropdown>
-          <TextField value={value} onChange={e => setValue(e.target.value)} autoFocus={true} />
         </Flexbox>
         <Button
           variant="primary"
diff --git a/components/pages/reports/FilterSelectForm.js b/components/pages/reports/FilterSelectForm.js
index 29493b08..49238e1c 100644
--- a/components/pages/reports/FilterSelectForm.js
+++ b/components/pages/reports/FilterSelectForm.js
@@ -1,13 +1,34 @@
 import { useState } from 'react';
 import FieldSelectForm from './FieldSelectForm';
 import FieldFilterForm from './FieldFilterForm';
+import { useApi } from 'hooks';
+import { Loading } from 'react-basics';
 
-export default function FilterSelectForm({ items, onSelect }) {
+function useValues(websiteId, type) {
+  const { get, useQuery } = useApi();
+  const { data, error, isLoading } = useQuery(
+    ['websites:values', websiteId, type],
+    () =>
+      get(`/websites/${websiteId}/values`, {
+        type,
+      }),
+    { enabled: !!(websiteId && type) },
+  );
+
+  return { data, error, isLoading };
+}
+
+export default function FilterSelectForm({ websiteId, items, onSelect }) {
   const [field, setField] = useState();
+  const { data, isLoading } = useValues(websiteId, field?.name);
 
   if (!field) {
     return <FieldSelectForm items={items} onSelect={setField} />;
   }
 
-  return <FieldFilterForm name={field.name} type={field.type} onSelect={onSelect} />;
+  if (isLoading) {
+    return <Loading />;
+  }
+
+  return <FieldFilterForm label={field?.label} type="string" values={data} onSelect={onSelect} />;
 }
diff --git a/components/pages/reports/insights/InsightsParameters.js b/components/pages/reports/insights/InsightsParameters.js
index 4ec60a9a..c6140d68 100644
--- a/components/pages/reports/insights/InsightsParameters.js
+++ b/components/pages/reports/insights/InsightsParameters.js
@@ -74,7 +74,11 @@ export function InsightsParameters() {
                   <FieldSelectForm items={fieldOptions} onSelect={handleAdd.bind(null, id)} />
                 )}
                 {id === 'filters' && (
-                  <FilterSelectForm items={fieldOptions} onSelect={handleAdd.bind(null, id)} />
+                  <FilterSelectForm
+                    websiteId={websiteId}
+                    items={fieldOptions}
+                    onSelect={handleAdd.bind(null, id)}
+                  />
                 )}
               </PopupForm>
             );
diff --git a/hooks/useFilters.js b/hooks/useFilters.js
index 51268382..0175e72a 100644
--- a/hooks/useFilters.js
+++ b/hooks/useFilters.js
@@ -1,6 +1,6 @@
 import { useMessages } from 'hooks';
 
-export function useFilters() {
+export function useFilters(type) {
   const { formatMessage, labels } = useMessages();
 
   const filters = {
@@ -27,7 +27,7 @@ export function useFilters() {
     uuid: ['eq'],
   };
 
-  return { filters, types };
+  return types[type]?.map(key => ({ type, value: key, label: filters[key] })) ?? [];
 }
 
 export default useFilters;
diff --git a/pages/api/websites/[id]/values.ts b/pages/api/websites/[id]/values.ts
new file mode 100644
index 00000000..b40fc262
--- /dev/null
+++ b/pages/api/websites/[id]/values.ts
@@ -0,0 +1,43 @@
+import { NextApiRequestQueryBody } from 'lib/types';
+import { canViewWebsite } from 'lib/auth';
+import { useAuth, useCors } from 'lib/middleware';
+import { NextApiResponse } from 'next';
+import { badRequest, methodNotAllowed, ok, unauthorized } from 'next-basics';
+import { EVENT_COLUMNS, SESSION_COLUMNS } from 'lib/constants';
+import { getValues } from 'queries';
+
+export interface WebsiteResetRequestQuery {
+  id: string;
+}
+
+export default async (
+  req: NextApiRequestQueryBody<WebsiteResetRequestQuery>,
+  res: NextApiResponse,
+) => {
+  await useCors(req, res);
+  await useAuth(req, res);
+
+  const { id: websiteId, type } = req.query;
+
+  if (req.method === 'GET') {
+    if (!SESSION_COLUMNS.includes(type as string) && !EVENT_COLUMNS.includes(type as string)) {
+      return badRequest(res);
+    }
+
+    if (!(await canViewWebsite(req.auth, websiteId))) {
+      return unauthorized(res);
+    }
+
+    const values = await getValues(websiteId, type as string);
+
+    return ok(
+      res,
+      values
+        .map(({ value }) => value)
+        .filter(n => n)
+        .sort(),
+    );
+  }
+
+  return methodNotAllowed(res);
+};
diff --git a/queries/analytics/stats/getActiveVisitors.ts b/queries/analytics/getActiveVisitors.ts
similarity index 84%
rename from queries/analytics/stats/getActiveVisitors.ts
rename to queries/analytics/getActiveVisitors.ts
index 8dcfd7c9..db3583eb 100644
--- a/queries/analytics/stats/getActiveVisitors.ts
+++ b/queries/analytics/getActiveVisitors.ts
@@ -17,10 +17,8 @@ async function relationalQuery(websiteId: string) {
     `
     select count(distinct session_id) x
     from website_event
-    join website 
-        on website_event.website_id = website.website_id
-    where website.website_id = {{websiteId::uuid}}
-    and website_event.created_at >= {{startAt}}
+    where website_id = {{websiteId::uuid}}
+    and created_at >= {{startAt}}
     `,
     { websiteId, startAt: subMinutes(new Date(), 5) },
   );
diff --git a/queries/analytics/stats/getRealtimeData.ts b/queries/analytics/getRealtimeData.ts
similarity index 93%
rename from queries/analytics/stats/getRealtimeData.ts
rename to queries/analytics/getRealtimeData.ts
index 98a5595b..8786ab13 100644
--- a/queries/analytics/stats/getRealtimeData.ts
+++ b/queries/analytics/getRealtimeData.ts
@@ -1,5 +1,5 @@
 import { md5 } from 'next-basics';
-import { getSessions, getEvents } from 'queries';
+import { getSessions, getEvents } from 'queries/index';
 import { EVENT_TYPE } from 'lib/constants';
 
 export async function getRealtimeData(websiteId, time) {
diff --git a/queries/analytics/getValues.ts b/queries/analytics/getValues.ts
new file mode 100644
index 00000000..1f088e3b
--- /dev/null
+++ b/queries/analytics/getValues.ts
@@ -0,0 +1,38 @@
+import prisma from 'lib/prisma';
+import clickhouse from 'lib/clickhouse';
+import { runQuery, CLICKHOUSE, PRISMA } from 'lib/db';
+
+export async function getValues(...args: [websiteId: string, column: string]) {
+  return runQuery({
+    [PRISMA]: () => relationalQuery(...args),
+    [CLICKHOUSE]: () => clickhouseQuery(...args),
+  });
+}
+
+async function relationalQuery(websiteId: string, column: string) {
+  const { rawQuery } = prisma;
+
+  return rawQuery(
+    `
+    select distinct ${column} as "value"
+    from website_event
+    inner join session
+      on session.session_id = website_event.session_id
+    where website_event.website_id = {{websiteId::uuid}}
+    `,
+    { websiteId },
+  );
+}
+
+async function clickhouseQuery(websiteId: string, column: string) {
+  const { rawQuery } = clickhouse;
+
+  return rawQuery(
+    `
+    select distinct ${column} as value
+    from website_event
+    where website_id = {websiteId:UUID}
+    `,
+    { websiteId },
+  );
+}
diff --git a/queries/analytics/stats/getWebsiteDateRange.ts b/queries/analytics/getWebsiteDateRange.ts
similarity index 100%
rename from queries/analytics/stats/getWebsiteDateRange.ts
rename to queries/analytics/getWebsiteDateRange.ts
diff --git a/queries/analytics/stats/getWebsiteStats.ts b/queries/analytics/getWebsiteStats.ts
similarity index 100%
rename from queries/analytics/stats/getWebsiteStats.ts
rename to queries/analytics/getWebsiteStats.ts
diff --git a/queries/analytics/reports/getInsights.ts b/queries/analytics/reports/getInsights.ts
index 4c47052b..2167aa2c 100644
--- a/queries/analytics/reports/getInsights.ts
+++ b/queries/analytics/reports/getInsights.ts
@@ -1,7 +1,7 @@
 import { CLICKHOUSE, PRISMA, runQuery } from 'lib/db';
 import prisma from 'lib/prisma';
 import clickhouse from 'lib/clickhouse';
-import { EVENT_TYPE } from 'lib/constants';
+import { EVENT_TYPE, SESSION_COLUMNS } from 'lib/constants';
 import { QueryFilters } from 'lib/types';
 
 export async function getInsights(
@@ -24,23 +24,29 @@ async function relationalQuery(
   }[]
 > {
   const { parseFilters, rawQuery } = prisma;
-  const { filterQuery, joinSession, params } = await parseFilters(websiteId, {
-    ...filters,
-    eventType: EVENT_TYPE.pageView,
-  });
+  const { filterQuery, joinSession, params } = await parseFilters(
+    websiteId,
+    {
+      ...filters,
+      eventType: EVENT_TYPE.pageView,
+    },
+    {
+      joinSession: !!fields.find(({ name }) => SESSION_COLUMNS.includes(name)),
+    },
+  );
 
   return rawQuery(
     `
-    select
-      url_path,
-      count(*) y
+    select 
+      ${parseFields(fields)}
     from website_event
       ${joinSession}
     where website_event.website_id = {{websiteId::uuid}}
       and website_event.created_at between {{startDate}} and {{endDate}}
       and website_event.event_type = {{eventType}}
       ${filterQuery}
-    group by 1
+    group by ${fields.map(({ name }) => name).join(',')}
+    order by 1 desc, 2 desc
     limit 500
     `,
     params,
@@ -87,7 +93,7 @@ function parseFields(fields) {
 
       return arr.concat(name);
     },
-    ['count(*) as views', 'count(distinct session_id) as visitors'],
+    ['count(*) as views', 'count(distinct website_event.session_id) as visitors'],
   );
 
   return query.join(',\n');
diff --git a/queries/index.js b/queries/index.js
index 474ab31e..b9c0a34d 100644
--- a/queries/index.js
+++ b/queries/index.js
@@ -21,7 +21,8 @@ export * from './analytics/sessions/getSessionMetrics';
 export * from './analytics/sessions/getSessions';
 export * from './analytics/sessions/getSessionStats';
 export * from './analytics/sessions/saveSessionData';
-export * from './analytics/stats/getActiveVisitors';
-export * from './analytics/stats/getRealtimeData';
-export * from './analytics/stats/getWebsiteDateRange';
-export * from './analytics/stats/getWebsiteStats';
+export * from './analytics/getActiveVisitors';
+export * from './analytics/getRealtimeData';
+export * from './analytics/getValues';
+export * from './analytics/getWebsiteDateRange';
+export * from './analytics/getWebsiteStats';