improvement(kb): add document filtering, select all, and React Query migration

apps/sim/app/api/knowledge/[id]/documents/route.test.ts

@@ -157,7 +157,7 @@ describe('Knowledge Base Documents API Route', () => {
       expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
         'kb-123',
         {
-          includeDisabled: false,
+          enabledFilter: undefined,
           search: undefined,
           limit: 50,
           offset: 0,
@@ -166,7 +166,7 @@ describe('Knowledge Base Documents API Route', () => {
       )
     })
 
-    it('should filter disabled documents by default', async () => {
+    it('should return documents with default filter', async () => {
       const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
       const { getDocuments } = await import('@/lib/knowledge/documents/service')
 
@@ -194,7 +194,7 @@ describe('Knowledge Base Documents API Route', () => {
       expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
         'kb-123',
         {
-          includeDisabled: false,
+          enabledFilter: undefined,
           search: undefined,
           limit: 50,
           offset: 0,
@@ -203,7 +203,7 @@ describe('Knowledge Base Documents API Route', () => {
       )
     })
 
-    it('should include disabled documents when requested', async () => {
+    it('should filter documents by enabled status when requested', async () => {
       const { checkKnowledgeBaseAccess } = await import('@/app/api/knowledge/utils')
       const { getDocuments } = await import('@/lib/knowledge/documents/service')
 
@@ -223,7 +223,7 @@ describe('Knowledge Base Documents API Route', () => {
         },
       })
 
-      const url = 'http://localhost:3000/api/knowledge/kb-123/documents?includeDisabled=true'
+      const url = 'http://localhost:3000/api/knowledge/kb-123/documents?enabledFilter=disabled'
       const req = new Request(url, { method: 'GET' }) as any
 
       const { GET } = await import('@/app/api/knowledge/[id]/documents/route')
@@ -233,7 +233,7 @@ describe('Knowledge Base Documents API Route', () => {
       expect(vi.mocked(getDocuments)).toHaveBeenCalledWith(
         'kb-123',
         {
-          includeDisabled: true,
+          enabledFilter: 'disabled',
           search: undefined,
           limit: 50,
           offset: 0,
@@ -361,8 +361,7 @@ describe('Knowledge Base Documents API Route', () => {
       expect(vi.mocked(createSingleDocument)).toHaveBeenCalledWith(
         validDocumentData,
         'kb-123',
-        expect.any(String),
-        'user-123'
+        expect.any(String)
       )
     })
 
@@ -470,8 +469,7 @@ describe('Knowledge Base Documents API Route', () => {
       expect(vi.mocked(createDocumentRecords)).toHaveBeenCalledWith(
         validBulkData.documents,
         'kb-123',
-        expect.any(String),
-        'user-123'
+        expect.any(String)
       )
       expect(vi.mocked(processDocumentsWithQueue)).toHaveBeenCalled()
     })

apps/sim/app/api/knowledge/[id]/documents/route.ts

@@ -5,6 +5,7 @@ import { z } from 'zod'
 import { getSession } from '@/lib/auth'
 import {
   bulkDocumentOperation,
+  bulkDocumentOperationByFilter,
   createDocumentRecords,
   createSingleDocument,
   getDocuments,
@@ -57,13 +58,20 @@ const BulkCreateDocumentsSchema = z.object({
   bulk: z.literal(true),
 })
 
-const BulkUpdateDocumentsSchema = z.object({
-  operation: z.enum(['enable', 'disable', 'delete']),
-  documentIds: z
-    .array(z.string())
-    .min(1, 'At least one document ID is required')
-    .max(100, 'Cannot operate on more than 100 documents at once'),
-})
+const BulkUpdateDocumentsSchema = z
+  .object({
+    operation: z.enum(['enable', 'disable', 'delete']),
+    documentIds: z
+      .array(z.string())
+      .min(1, 'At least one document ID is required')
+      .max(100, 'Cannot operate on more than 100 documents at once')
+      .optional(),
+    selectAll: z.boolean().optional(),
+    enabledFilter: z.enum(['all', 'enabled', 'disabled']).optional(),
+  })
+  .refine((data) => data.selectAll || (data.documentIds && data.documentIds.length > 0), {
+    message: 'Either selectAll must be true or documentIds must be provided',
+  })
 
 export async function GET(req: NextRequest, { params }: { params: Promise<{ id: string }> }) {
   const requestId = randomUUID().slice(0, 8)
@@ -90,21 +98,25 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
     }
 
     const url = new URL(req.url)
-    const includeDisabled = url.searchParams.get('includeDisabled') === 'true'
+    const enabledFilter = url.searchParams.get('enabledFilter') as
+      | 'all'
+      | 'enabled'
+      | 'disabled'
+      | null
     const search = url.searchParams.get('search') || undefined
     const limit = Number.parseInt(url.searchParams.get('limit') || '50')
     const offset = Number.parseInt(url.searchParams.get('offset') || '0')
     const sortByParam = url.searchParams.get('sortBy')
     const sortOrderParam = url.searchParams.get('sortOrder')
 
-    // Validate sort parameters
     const validSortFields: DocumentSortField[] = [
       'filename',
       'fileSize',
       'tokenCount',
       'chunkCount',
       'uploadedAt',
       'processingStatus',
+      'enabled',
     ]
     const validSortOrders: SortOrder[] = ['asc', 'desc']
 
@@ -120,7 +132,7 @@ export async function GET(req: NextRequest, { params }: { params: Promise<{ id:
     const result = await getDocuments(
       knowledgeBaseId,
       {
-        includeDisabled,
+        enabledFilter: enabledFilter || undefined,
         search,
         limit,
         offset,
@@ -190,8 +202,7 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
         const createdDocuments = await createDocumentRecords(
           validatedData.documents,
           knowledgeBaseId,
-          requestId,
-          userId
+          requestId
         )
 
         logger.info(
@@ -250,16 +261,10 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
         throw validationError
       }
     } else {
-      // Handle single document creation
       try {
         const validatedData = CreateDocumentSchema.parse(body)
 
-        const newDocument = await createSingleDocument(
-          validatedData,
-          knowledgeBaseId,
-          requestId,
-          userId
-        )
+        const newDocument = await createSingleDocument(validatedData, knowledgeBaseId, requestId)
 
         try {
           const { PlatformEvents } = await import('@/lib/core/telemetry')
@@ -294,7 +299,6 @@ export async function POST(req: NextRequest, { params }: { params: Promise<{ id:
   } catch (error) {
     logger.error(`[${requestId}] Error creating document`, error)
 
-    // Check if it's a storage limit error
     const errorMessage = error instanceof Error ? error.message : 'Failed to create document'
     const isStorageLimitError =
       errorMessage.includes('Storage limit exceeded') || errorMessage.includes('storage limit')
@@ -331,16 +335,22 @@ export async function PATCH(req: NextRequest, { params }: { params: Promise<{ id
 
     try {
       const validatedData = BulkUpdateDocumentsSchema.parse(body)
-      const { operation, documentIds } = validatedData
+      const { operation, documentIds, selectAll, enabledFilter } = validatedData
 
       try {
-        const result = await bulkDocumentOperation(
-          knowledgeBaseId,
-          operation,
-          documentIds,
-          requestId,
-          session.user.id
-        )
+        let result
+        if (selectAll) {
+          result = await bulkDocumentOperationByFilter(
+            knowledgeBaseId,
+            operation,
+            enabledFilter,
+            requestId
+          )
+        } else if (documentIds && documentIds.length > 0) {
+          result = await bulkDocumentOperation(knowledgeBaseId, operation, documentIds, requestId)
+        } else {
+          return NextResponse.json({ error: 'No documents specified' }, { status: 400 })
+        }
 
         return NextResponse.json({
           success: true,

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/components/edit-chunk-modal/edit-chunk-modal.tsx

@@ -61,6 +61,7 @@ export function EditChunkModal({
   const [showUnsavedChangesAlert, setShowUnsavedChangesAlert] = useState(false)
   const [pendingNavigation, setPendingNavigation] = useState<(() => void) | null>(null)
   const [tokenizerOn, setTokenizerOn] = useState(false)
+  const [hoveredTokenIndex, setHoveredTokenIndex] = useState<number | null>(null)
   const textareaRef = useRef<HTMLTextAreaElement>(null)
 
   const error = mutationError?.message ?? null
@@ -254,6 +255,8 @@ export function EditChunkModal({
                         style={{
                           backgroundColor: getTokenBgColor(index),
                         }}
+                        onMouseEnter={() => setHoveredTokenIndex(index)}
+                        onMouseLeave={() => setHoveredTokenIndex(null)}
                       >
                         {token}
                       </span>
@@ -281,6 +284,11 @@ export function EditChunkModal({
                 <div className='flex items-center gap-[8px]'>
                   <span className='text-[12px] text-[var(--text-secondary)]'>Tokenizer</span>
                   <Switch checked={tokenizerOn} onCheckedChange={setTokenizerOn} />
+                  {tokenizerOn && hoveredTokenIndex !== null && (
+                    <span className='text-[12px] text-[var(--text-tertiary)]'>
+                      Token #{hoveredTokenIndex + 1}
+                    </span>
+                  )}
                 </div>
                 <span className='text-[12px] text-[var(--text-secondary)]'>
                   {tokenCount.toLocaleString()}

apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx

@@ -36,6 +36,7 @@ import {
 import { Input } from '@/components/ui/input'
 import { SearchHighlight } from '@/components/ui/search-highlight'
 import { Skeleton } from '@/components/ui/skeleton'
+import { formatAbsoluteDate, formatRelativeTime } from '@/lib/core/utils/formatting'
 import type { ChunkData } from '@/lib/knowledge/types'
 import {
   ChunkContextMenu,
@@ -58,55 +59,6 @@ import {
 
 const logger = createLogger('Document')
 
-/**
- * Formats a date string to relative time (e.g., "2h ago", "3d ago")
- */
-function formatRelativeTime(dateString: string): string {
-  const date = new Date(dateString)
-  const now = new Date()
-  const diffInSeconds = Math.floor((now.getTime() - date.getTime()) / 1000)
-
-  if (diffInSeconds < 60) {
-    return 'just now'
-  }
-  if (diffInSeconds < 3600) {
-    const minutes = Math.floor(diffInSeconds / 60)
-    return `${minutes}m ago`
-  }
-  if (diffInSeconds < 86400) {
-    const hours = Math.floor(diffInSeconds / 3600)
-    return `${hours}h ago`
-  }
-  if (diffInSeconds < 604800) {
-    const days = Math.floor(diffInSeconds / 86400)
-    return `${days}d ago`
-  }
-  if (diffInSeconds < 2592000) {
-    const weeks = Math.floor(diffInSeconds / 604800)
-    return `${weeks}w ago`
-  }
-  if (diffInSeconds < 31536000) {
-    const months = Math.floor(diffInSeconds / 2592000)
-    return `${months}mo ago`
-  }
-  const years = Math.floor(diffInSeconds / 31536000)
-  return `${years}y ago`
-}
-
-/**
- * Formats a date string to absolute format for tooltip display
- */
-function formatAbsoluteDate(dateString: string): string {
-  const date = new Date(dateString)
-  return date.toLocaleDateString('en-US', {
-    year: 'numeric',
-    month: 'short',
-    day: 'numeric',
-    hour: '2-digit',
-    minute: '2-digit',
-  })
-}
-
 interface DocumentProps {
   knowledgeBaseId: string
   documentId: string
@@ -304,7 +256,6 @@ export function Document({
 
   const [searchQuery, setSearchQuery] = useState('')
   const [debouncedSearchQuery, setDebouncedSearchQuery] = useState('')
-  const [isSearching, setIsSearching] = useState(false)
 
   const {
     chunks: initialChunks,
@@ -344,7 +295,6 @@ export function Document({
     const handler = setTimeout(() => {
       startTransition(() => {
         setDebouncedSearchQuery(searchQuery)
-        setIsSearching(searchQuery.trim().length > 0)
       })
     }, 200)
 
@@ -353,6 +303,7 @@ export function Document({
     }
   }, [searchQuery])
 
+  const isSearching = debouncedSearchQuery.trim().length > 0
   const showingSearch = isSearching && searchQuery.trim().length > 0 && searchResults.length > 0
   const SEARCH_PAGE_SIZE = 50
   const maxSearchPages = Math.ceil(searchResults.length / SEARCH_PAGE_SIZE)