diff --git a/packages/client/src/components/settings/FileTable.tsx b/packages/client/src/components/settings/FileTable.tsx index 4c9b45824..e65e77a4e 100644 --- a/packages/client/src/components/settings/FileTable.tsx +++ b/packages/client/src/components/settings/FileTable.tsx @@ -13,6 +13,10 @@ import { Table, Typography, useNotification, + Stack, + Select, + MenuItem, + Box, } from '@semoss/ui'; import { Add, Delete, SimCardDownload } from '@mui/icons-material'; import { usePixel, useRootStore } from '@/hooks'; @@ -57,6 +61,12 @@ const StyledIcon = styled(Add)(({ theme }) => ({ const StyledFileTable = styled(Table)({ backgroundColor: 'white' }); +const splittingOptions = [ + { label: 'Tokens', value: 'Tokens' }, + { label: 'Recursive', value: 'Recursive' }, + { label: 'Semantic', value: 'Semantic' }, +]; + interface FileTableProps { /** * Id of the vector engine @@ -122,6 +132,8 @@ export const FileTable = (props: FileTableProps) => { }, ]; + const [selectedSplitOptions, setSelectedSplitOptions] = useState('Tokens'); + //grabbing ID out of props const { id } = props; @@ -227,7 +239,7 @@ export const FileTable = (props: FileTableProps) => { // Embedding the File const response = await monolithStore.runQuery(` - CreateEmbeddingsFromDocuments( engine= "${id}", filePaths= [${fileLocations}]) + CreateEmbeddingsFromDocuments( engine= "${id}", filePaths= [${fileLocations}], splitOptions="${selectedSplitOptions}") `); const { output, operationType } = response.pixelReturn[0]; @@ -622,23 +634,55 @@ export const FileTable = (props: FileTableProps) => { rules={{}} render={({ field }) => { return ( - { - field.onChange(newValues); - }} - /> + + + + + { + field.onChange(newValues); + }} + /> + ); }} /> diff --git a/packages/client/src/pages/import/ImportConnectionPage.tsx b/packages/client/src/pages/import/ImportConnectionPage.tsx index c9c668b75..d73245851 100644 --- a/packages/client/src/pages/import/ImportConnectionPage.tsx +++ b/packages/client/src/pages/import/ImportConnectionPage.tsx @@ -6,6 +6,19 @@ import { useRootStore } from '@/hooks'; import { useStepper } from '@/hooks'; +interface VectorDatabaseFields { + NAME: string; + VECTOR_TYPE: string; + EMBEDDER_ENGINE_ID: string; + INDEX_CLASSES: string; + CHUNKING_STRATEGY: string; + CONTENT_LENGTH: number; + CONTENT_OVERLAP: number; + KEEP_INPUT_OUTPUT: string; + DISTANCE_METHOD: string; + SPLITTING_OPTIONS?: string; +} + const StyledBox = styled(Box)(({ theme }) => ({ boxShadow: '0px 5px 22px 0px rgba(0, 0, 0, 0.06)', width: '100%', @@ -35,7 +48,7 @@ export const ImportConnectionPage = () => { const formSubmit = async (values: { type: 'VECTOR' | 'STORAGE' | 'MODEL' | 'FUNCTION' | 'UPLOAD'; name: string; - fields: unknown[]; + fields: VectorDatabaseFields; secondaryFields?: unknown[]; }) => { // let pixel = ''; // 'VECTOR' | 'STORAGE' | 'MODEL' | 'FUNCTION' | 'UPLOAD' @@ -113,10 +126,13 @@ export const ImportConnectionPage = () => { return; } else if (values.type === 'VECTOR') { /** Vector Database: START */ + + const { SPLITTING_OPTIONS, ...filteredFields } = values.fields; + const pixel = ` CreateVectorDatabaseEngine ( database=["${values.name}"], - conDetails=[${JSON.stringify(values.fields)}] + conDetails=[${JSON.stringify(filteredFields)}] ) ; `; @@ -146,7 +162,8 @@ export const ImportConnectionPage = () => { const secondaryPixel = `CreateEmbeddingsFromDocuments( engine="${output.database_id}", - filePaths=["${upload[0].fileLocation}"] + filePaths=["${upload[0].fileLocation}"], + splitOptions="${SPLITTING_OPTIONS}" );`; monolithStore.runQuery(secondaryPixel).then((response) => { diff --git a/packages/client/src/pages/import/import.constants.ts b/packages/client/src/pages/import/import.constants.ts index 770917bbc..14d3259f9 100644 --- a/packages/client/src/pages/import/import.constants.ts +++ b/packages/client/src/pages/import/import.constants.ts @@ -6742,6 +6742,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DISTANCE_METHOD', label: 'Distance Method', @@ -6983,6 +7009,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DIMENSION_SIZE', label: 'Embedding Dimension Size', @@ -7234,6 +7286,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DISTANCE_METHOD', label: 'Distance Method', @@ -7464,6 +7542,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DISTANCE_METHOD', label: 'Distance Method', @@ -7729,6 +7833,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DISTANCE_METHOD', label: 'Distance Method', @@ -8054,6 +8184,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DISTANCE_METHOD', label: 'Distance Method', @@ -8539,6 +8695,32 @@ export const CONNECTION_OPTIONS = { secondary: true, rules: {}, }, + { + fieldName: 'SPLITTING_OPTIONS', + label: 'Splitting Options', + defaultValue: 'Tokens', + options: { + component: 'select', + options: [ + { + display: 'Tokens', + value: 'Tokens', + }, + { + display: 'Recursive', + value: 'Recursive', + }, + { + display: 'Semantic', + value: 'Semantic', + }, + ], + }, + disabled: false, + rules: { required: false }, + advanced: true, + helperText: '', + }, { fieldName: 'DISTANCE_METHOD', label: 'Distance Method',