Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(data-warehouse): Added incremental syncs for postgres #23145

Merged
merged 11 commits into from
Jun 24, 2024
Merged

feat(data-warehouse): Added incremental syncs for postgres #23145

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
3c18d54
Added incremental syncs for postgres
Gilbert09 Jun 21, 2024
4e16758
Merge branch 'master' into tom/incremental-postgres
Gilbert09 Jun 21, 2024
2d9264b
Tests
Gilbert09 Jun 21, 2024
bd001a7
Fixed mypy
Gilbert09 Jun 21, 2024
c90dc77
Added extra protection on the sync form
Gilbert09 Jun 21, 2024
e1cec02
Set the correct write disposition for SQL
Gilbert09 Jun 21, 2024
0c4059b
PR fixes
Gilbert09 Jun 24, 2024
c7687e8
Merge branch 'master' into tom/incremental-postgres
Gilbert09 Jun 24, 2024
b9d5841
Fixed tests
Gilbert09 Jun 24, 2024
e9fe396
Fixed tests
Gilbert09 Jun 24, 2024
fa6df18
Fixed tests
Gilbert09 Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions frontend/src/lib/api.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ import {
RolesListParams,
RoleType,
ScheduledChangeType,
SchemaIncrementalFieldsResponse,
SearchListParams,
SearchResponse,
SessionRecordingPlaylistType,
Expand Down Expand Up @@ -2081,6 +2082,9 @@ const api = {
async resync(schemaId: ExternalDataSourceSchema['id']): Promise<void> {
await new ApiRequest().externalDataSourceSchema(schemaId).withAction('resync').create()
},
async incremental_fields(schemaId: ExternalDataSourceSchema['id']): Promise<SchemaIncrementalFieldsResponse> {
return await new ApiRequest().externalDataSourceSchema(schemaId).withAction('incremental_fields').create()
},
},

dataWarehouseViewLinks: {
Expand Down
8 changes: 7 additions & 1 deletion frontend/src/lib/lemon-ui/LemonRadio/LemonRadio.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ export interface LemonRadioProps<T extends React.Key> {
onChange: (newValue: T) => void
options: LemonRadioOption<T>[]
className?: string
radioPosition?: 'center' | 'top'
}

/** Single choice radio. */
Expand All @@ -23,6 +24,7 @@ export function LemonRadio<T extends React.Key>({
onChange,
options,
className,
radioPosition,
}: LemonRadioProps<T>): JSX.Element {
return (
<div className={clsx('flex flex-col gap-2 font-medium', className)}>
Expand All @@ -32,7 +34,11 @@ export function LemonRadio<T extends React.Key>({
key={value}
className={clsx(
'grid items-center gap-x-2 grid-cols-[min-content_auto] text-sm',
disabledReason ? 'text-muted cursor-not-allowed' : 'cursor-pointer'
disabledReason ? 'text-muted cursor-not-allowed' : 'cursor-pointer',
{
'items-baseline': radioPosition === 'top',
'items-center': radioPosition === 'center' || !radioPosition,
}
)}
>
<input
Expand Down
187 changes: 104 additions & 83 deletions frontend/src/scenes/data-warehouse/external/forms/PostgresSchemaForm.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,98 +1,119 @@
import { LemonSelect, LemonSelectOptionLeaf, LemonSwitch, LemonTable, Link } from '@posthog/lemon-ui'
import { LemonButton, LemonCheckbox, LemonModal, LemonTable } from '@posthog/lemon-ui'
import { useActions, useValues } from 'kea'
import { useState } from 'react'

import { ExternalDataSourceSyncSchema } from '~/types'

import { sourceWizardLogic } from '../../new/sourceWizardLogic'

const syncTypesToOptions = (
schema: ExternalDataSourceSyncSchema
): LemonSelectOptionLeaf<ExternalDataSourceSyncSchema['sync_type']>[] => {
const options: LemonSelectOptionLeaf<ExternalDataSourceSyncSchema['sync_type']>[] = []

if (schema.sync_types.full_refresh) {
options.push({ value: 'full_refresh', label: 'Full refresh' })
}

if (schema.sync_types.incremental) {
options.push({ value: 'incremental', label: 'Incremental' })
}

return options
}
import { SyncMethodForm } from './SyncMethodForm'

export default function PostgresSchemaForm(): JSX.Element {
const { toggleSchemaShouldSync, updateSchemaSyncType } = useActions(sourceWizardLogic)
const { toggleSchemaShouldSync, openSyncMethodModal } = useActions(sourceWizardLogic)
const { databaseSchema } = useValues(sourceWizardLogic)
const [toggleAllState, setToggleAllState] = useState(false)

const toggleAllSwitches = (): void => {
databaseSchema.forEach((schema) => {
toggleSchemaShouldSync(schema, toggleAllState)
})

setToggleAllState(!toggleAllState)
}

return (
<div className="flex flex-col gap-2">
<div>
<LemonTable
emptyState="No schemas found"
dataSource={databaseSchema}
columns={[
{
title: 'Table',
key: 'table',
render: function RenderTable(_, schema) {
return schema.table
<>
<div className="flex flex-col gap-2">
<div>
<LemonTable
emptyState="No schemas found"
dataSource={databaseSchema}
columns={[
{
width: 0,
key: 'enabled',
render: (_, schema) => {
return (
<LemonCheckbox
checked={schema.should_sync}
onChange={(checked) => {
toggleSchemaShouldSync(schema, checked)
}}
/>
)
},
},
},
{
title: (
<>
<span>Sync</span>
<Link
className="ml-2 w-[60px] overflow-visible"
onClick={() => toggleAllSwitches()}
>
{toggleAllState ? 'Enable' : 'Disable'} all
</Link>
</>
),
key: 'should_sync',
render: function RenderShouldSync(_, schema) {
return (
<LemonSwitch
checked={schema.should_sync}
onChange={(checked) => {
toggleSchemaShouldSync(schema, checked)
}}
/>
)
{
title: 'Table',
key: 'table',
render: function RenderTable(_, schema) {
return schema.table
},
},
},
{
key: 'sync_type',
title: 'Sync type',
tooltip:
'Full refresh will refresh the full table on every sync, whereas incremental will only sync new and updated rows since the last sync',
render: (_, schema) => {
const options = syncTypesToOptions(schema)
{
key: 'sync_type',
title: 'Sync method',
align: 'right',
tooltip:
'Full refresh will refresh the full table on every sync, whereas incremental will only sync new and updated rows since the last sync',
render: (_, schema) => {
if (!schema.sync_type) {
return (
<div className="justify-end flex">
<LemonButton
className="my-1"
type="primary"
onClick={() => openSyncMethodModal(schema)}
>
Set up
</LemonButton>
</div>
)
}

return (
<LemonSelect
options={options}
value={schema.sync_type}
onChange={(newValue) => updateSchemaSyncType(schema, newValue)}
/>
)
return (
<div className="justify-end flex">
<LemonButton
className="my-1"
size="small"
type="secondary"
onClick={() => openSyncMethodModal(schema)}
>
{schema.sync_type === 'full_refresh' ? 'Full refresh' : 'Incremental'}
</LemonButton>
</div>
)
},
},
},
]}
/>
]}
/>
</div>
</div>
</div>
<SyncMethodModal />
</>
)
}

const SyncMethodModal = (): JSX.Element => {
const { cancelSyncMethodModal, updateSchemaSyncType, toggleSchemaShouldSync } = useActions(sourceWizardLogic)
const { syncMethodModalOpen, currentSyncMethodModalSchema } = useValues(sourceWizardLogic)

if (!currentSyncMethodModalSchema) {
return <></>
}

return (
<LemonModal
title={`Sync method for ${currentSyncMethodModalSchema.table}`}
isOpen={syncMethodModalOpen}
onClose={cancelSyncMethodModal}
>
<SyncMethodForm
schema={currentSyncMethodModalSchema}
onClose={cancelSyncMethodModal}
onSave={(syncType, incrementalField, incrementalFieldType) => {
if (syncType === 'incremental') {
updateSchemaSyncType(
currentSyncMethodModalSchema,
syncType,
incrementalField,
incrementalFieldType
)
} else {
updateSchemaSyncType(currentSyncMethodModalSchema, syncType ?? null, null, null)
}

toggleSchemaShouldSync(currentSyncMethodModalSchema, true)
cancelSyncMethodModal()
}}
/>
</LemonModal>
)
}
Loading
Loading