feat(cli): use multipart uploads for larger files

.changeset/every-rockets-tie.md

@@ -0,0 +1,5 @@
+---
+"@mixedbread/cli": minor
+---
+
+Support multipart uploads and use it by default for larger files

.github/workflows/prerelease.yml

@@ -4,9 +4,9 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: 'Prerelease tag (e.g., beta, alpha, rc)'
+        description: "Prerelease tag (e.g., beta, alpha, rc)"
         required: true
-        default: 'beta'
+        default: "beta"
         type: choice
         options:
           - beta
@@ -37,8 +37,8 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 24
-          cache: 'pnpm'
-          registry-url: 'https://registry.npmjs.org'
+          cache: "pnpm"
+          registry-url: "https://registry.npmjs.org"
 
       - name: Install dependencies
         run: pnpm install --frozen-lockfile
@@ -60,7 +60,7 @@ jobs:
           git commit -m "chore: version packages for ${{ inputs.tag }} release" || echo "No changes to commit"
 
       - name: Publish prerelease packages
-        run: pnpm changeset publish --tag ${{ inputs.tag }}
+        run: pnpm changeset publish
         env:
           NPM_CONFIG_PROVENANCE: true
 
@@ -72,4 +72,5 @@ jobs:
           pnpm changeset pre exit || echo "Not in pre mode"
           git add .
           git commit -m "chore: exit prerelease mode" || echo "No changes to commit"
-          git push || echo "No changes to push"
+          git push || echo "No changes to push"
+

packages/cli/package.json

@@ -50,7 +50,7 @@
   },
   "dependencies": {
     "@clack/prompts": "^1.0.1",
-    "@mixedbread/sdk": "^0.51.0",
+    "@mixedbread/sdk": "^0.57.0",
     "@pnpm/tabtab": "^0.5.4",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",

packages/cli/src/commands/store/sync.ts

@@ -5,6 +5,7 @@ import { z } from "zod";
 import { createClient } from "../../utils/client";
 import { warnContextualizationDeprecated } from "../../utils/deprecation";
 import { getGitInfo } from "../../utils/git";
+import type { MultipartUploadOptions } from "../../utils/upload";
 import {
   addGlobalOptions,
   extendGlobalOptions,
@@ -43,6 +44,19 @@ const SyncStoreSchema = extendGlobalOptions({
     .max(200, { error: '"parallel" must be less than or equal to 200' })
     .optional()
     .default(100),
+  multipartThreshold: z.coerce
+    .number({ error: '"multipart-threshold" must be a number' })
+    .min(5, { error: '"multipart-threshold" must be at least 5 MB' })
+    .optional(),
+  multipartPartSize: z.coerce
+    .number({ error: '"multipart-part-size" must be a number' })
+    .min(5, { error: '"multipart-part-size" must be at least 5 MB' })
+    .optional(),
+  multipartConcurrency: z.coerce
+    .number({ error: '"multipart-concurrency" must be a number' })
+    .int({ error: '"multipart-concurrency" must be an integer' })
+    .min(1, { error: '"multipart-concurrency" must be at least 1' })
+    .optional(),
 });
 
 export function createSyncCommand(): Command {
@@ -71,6 +85,18 @@ export function createSyncCommand(): Command {
       )
       .option("--metadata <json>", "Additional metadata for files")
       .option("--parallel <n>", "Number of concurrent operations (1-200)")
+      .option(
+        "--multipart-threshold <mb>",
+        "File size threshold in MB to trigger multipart upload",
+      )
+      .option(
+        "--multipart-part-size <mb>",
+        "Size of each part in MB for multipart upload",
+      )
+      .option(
+        "--multipart-concurrency <n>",
+        "Number of concurrent part uploads for multipart upload",
+      )
   );
 
   command.action(async (nameOrId: string, patterns: string[]) => {
@@ -188,12 +214,27 @@ export function createSyncCommand(): Command {
         log.success("Auto-proceeding with --yes flag");
       }
 
+      // Build multipart upload options
+      const MB = 1024 * 1024;
+      const multipartUpload: MultipartUploadOptions = {
+        ...(parsedOptions.multipartThreshold != null && {
+          threshold: parsedOptions.multipartThreshold * MB,
+        }),
+        ...(parsedOptions.multipartPartSize != null && {
+          partSize: parsedOptions.multipartPartSize * MB,
+        }),
+        ...(parsedOptions.multipartConcurrency != null && {
+          concurrency: parsedOptions.multipartConcurrency,
+        }),
+      };
+
       // Execute changes
       const syncResults = await executeSyncChanges(client, store.id, analysis, {
         strategy: parsedOptions.strategy,
         metadata: additionalMetadata,
         gitInfo: gitInfo.isRepo ? gitInfo : undefined,
         parallel: parsedOptions.parallel,
+        multipartUpload,
       });
 
       // Display summary

packages/cli/src/commands/store/upload.ts

@@ -18,7 +18,11 @@ import { uploadFromManifest } from "../../utils/manifest";
 import { validateMetadata } from "../../utils/metadata";
 import { formatBytes, formatCountWithSuffix } from "../../utils/output";
 import { checkExistingFiles, resolveStore } from "../../utils/store";
-import { type FileToUpload, uploadFilesInBatch } from "../../utils/upload";
+import {
+  type FileToUpload,
+  type MultipartUploadOptions,
+  uploadFilesInBatch,
+} from "../../utils/upload";
 
 const UploadStoreSchema = extendGlobalOptions({
   nameOrId: z.string().min(1, { error: '"name-or-id" is required' }),
@@ -41,6 +45,19 @@ const UploadStoreSchema = extendGlobalOptions({
     .optional(),
   unique: z.boolean().optional(),
   manifest: z.string().optional(),
+  multipartThreshold: z.coerce
+    .number({ error: '"multipart-threshold" must be a number' })
+    .min(5, { error: '"multipart-threshold" must be at least 5 MB' })
+    .optional(),
+  multipartPartSize: z.coerce
+    .number({ error: '"multipart-part-size" must be a number' })
+    .min(5, { error: '"multipart-part-size" must be at least 5 MB' })
+    .optional(),
+  multipartConcurrency: z.coerce
+    .number({ error: '"multipart-concurrency" must be a number' })
+    .int({ error: '"multipart-concurrency" must be an integer' })
+    .min(1, { error: '"multipart-concurrency" must be at least 1' })
+    .optional(),
 });
 
 export interface UploadOptions extends GlobalOptions {
@@ -51,6 +68,9 @@ export interface UploadOptions extends GlobalOptions {
   parallel?: number;
   unique?: boolean;
   manifest?: string;
+  multipartThreshold?: number;
+  multipartPartSize?: number;
+  multipartConcurrency?: number;
 }
 
 export function createUploadCommand(): Command {
@@ -76,6 +96,18 @@ export function createUploadCommand(): Command {
         false
       )
       .option("--manifest <file>", "Upload using manifest file")
+      .option(
+        "--multipart-threshold <mb>",
+        "File size threshold in MB to trigger multipart upload",
+      )
+      .option(
+        "--multipart-part-size <mb>",
+        "Size of each part in MB for multipart upload",
+      )
+      .option(
+        "--multipart-concurrency <n>",
+        "Number of concurrent part uploads for multipart upload",
+      )
   );
 
   command.action(async (nameOrId: string, patterns: string[]) => {
@@ -102,13 +134,27 @@ export function createUploadCommand(): Command {
       activeSpinner.stop("Upload initialized");
       activeSpinner = null;
 
+      const MB = 1024 * 1024;
+      const multipartUpload: MultipartUploadOptions = {
+        ...(parsedOptions.multipartThreshold != null && {
+          threshold: parsedOptions.multipartThreshold * MB,
+        }),
+        ...(parsedOptions.multipartPartSize != null && {
+          partSize: parsedOptions.multipartPartSize * MB,
+        }),
+        ...(parsedOptions.multipartConcurrency != null && {
+          concurrency: parsedOptions.multipartConcurrency,
+        }),
+      };
+
       // Handle manifest file upload
       if (parsedOptions.manifest) {
         return await uploadFromManifest(
           client,
           store.id,
           parsedOptions.manifest,
-          parsedOptions
+          parsedOptions,
+          multipartUpload
         );
       }
 
@@ -212,6 +258,7 @@ export function createUploadCommand(): Command {
         unique: parsedOptions.unique || false,
         existingFiles,
         parallel,
+        multipartUpload,
       });
     } catch (error) {
       activeSpinner?.stop();

packages/cli/src/utils/manifest.ts

@@ -12,7 +12,11 @@ import { log, spinner } from "./logger";
 import { validateMetadata } from "./metadata";
 import { formatBytes, formatCountWithSuffix } from "./output";
 import { checkExistingFiles } from "./store";
-import { type FileToUpload, uploadFilesInBatch } from "./upload";
+import {
+  type FileToUpload,
+  type MultipartUploadOptions,
+  uploadFilesInBatch,
+} from "./upload";
 
 // Manifest file schema
 const ManifestFileEntrySchema = z.object({
@@ -42,7 +46,8 @@ export async function uploadFromManifest(
   client: Mixedbread,
   storeIdentifier: string,
   manifestPath: string,
-  options: UploadOptions
+  options: UploadOptions,
+  multipartUpload?: MultipartUploadOptions
 ) {
   console.log(chalk.bold(`Loading manifest from: ${manifestPath}`));
 
@@ -199,6 +204,7 @@ export async function uploadFromManifest(
       existingFiles,
       parallel: options.parallel ?? config.defaults.upload.parallel ?? 100,
       showStrategyPerFile: true,
+      multipartUpload,
     });
   } catch (error) {
     if (error instanceof z.ZodError) {