> ## Documentation Index
> Fetch the complete documentation index at: https://docs.portkey.ai/docs/llms.txt
> Use this file to discover all available pages before exploring further.

# Batches

> Perform batch inference with Azure OpenAI

With Portkey, you can perform [Azure OpenAI Batch Inference](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/batch?tabs=global-batch%2Cstandard-input%2Cpython-secure\&pivots=rest-api) operations.
This is the most efficient way to

* Test your data with different foundation models
* Perform A/B testing with different foundation models
* Perform batch inference with different foundation models

> [!NOTE]
>
> Portkey supports **two modes** on Azure OpenAI:
>
> * **[Provider Batch API](#using-azure-openai-batch-api-through-portkey)** (cheapest, completion\_window: `24h`, `48h`, etc.)
> * **[Portkey Batch API](https://docs.portkey.ai/docs/product/ai-gateway/batches.md#portkey-batch-api-mode)** (fast, provider-agnostic, completion\_window: `immediate`)

## Using Azure OpenAI Batch API through Portkey

### Create Batch Job

```python Python theme={"system"}
from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

start_batch_response = portkey.batches.create(
  input_file_id="file_id", # file id of the input file
  endpoint="endpoint", # ex: /v1/chat/completions
  completion_window="completion_window", # ex: 24h
  metadata={} # metadata for the batch
)

print(start_batch_response)
```

```javascript Typescript theme={"system"}
import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const startBatch = async () => {
  const startBatchResponse = await portkey.batches.create({
    input_file_id: "file_id", // file id of the input file
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {} // metadata for the batch
  });

  console.log(startBatchResponse);
}

await startBatch();

```

```bash curl theme={"system"}
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider' \
--header 'Content-Type: application/json' \
--data '{
    "input_file_id": "<file_id>",
    "endpoint": "<endpoint>",
    "completion_window": "<completion_window>",
    "metadata": {}
}'
```

```javascript OpenAI NodeJS theme={"system"}
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const startBatch = async () => {
  const startBatchResponse = await openai.batches.create({
    input_file_id: "file_id", // file id of the input file
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {} // metadata for the batch
  });

  console.log(startBatchResponse);
}

await startBatch();
```

```python OpenAI Python theme={"system"}
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

start_batch_response = openai.batches.create(
  input_file_id="file_id", # file id of the input file
  endpoint="endpoint", # ex: /v1/chat/completions
  completion_window="completion_window", # ex: 24h
  metadata={} # metadata for the batch
)

print(start_batch_response)
```

### Create Batch Job with Blob Storage

```python Python theme={"system"}
from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

start_batch_response = portkey.batches.create(
  endpoint="endpoint",  # ex: /v1/chat/completions
  completion_window="completion_window",  # ex: 24h
  metadata={},  # metadata for the batch
  input_blob="<blob_url>",
  output_folder={
    "url": "<output_blob_folder>" # both error file and output file will be saved in this folder
  }
)

print(start_batch_response)
```

```javascript Typescript theme={"system"}
import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const startBatch = async () => {
  const startBatchResponse = await portkey.batches.create({
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {}, // metadata for the batch
    input_blob: "<blob_url>",
    output_folder: {
      url: "<output_blob_folder>" // both error file and output file will be saved in this folder
    }
  });

  console.log(startBatchResponse);
}

await startBatch();
```

```bash curl theme={"system"}
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider' \
--header 'Content-Type: application/json' \
--data '{
    "endpoint": "<endpoint>",
    "completion_window": "<completion_window>",
    "metadata": {},
    "input_blob": "<blob_url>",
    "output_folder": {
      "url": "<output_blob_folder>" # both error file and output file will be saved in this folder
    }
}'
```

```javascript OpenAI NodeJS theme={"system"}
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const startBatch = async () => {
  const startBatchResponse = await openai.batches.create({
    endpoint: "endpoint", // ex: /v1/chat/completions
    completion_window: "completion_window", // ex: 24h
    metadata: {}, // metadata for the batch
    extra_body: {
      input_blob: "<blob_url>",
      output_folder: {
        url: "<output_blob_folder>" // both error file and output file will be saved in this folder
      }
    }
  });

  console.log(startBatchResponse);
}

await startBatch();
```

```python OpenAI Python theme={"system"}
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

start_batch_response = openai.batches.create(
  endpoint="endpoint",  # ex: /v1/chat/completions
  completion_window="completion_window",  # ex: 24h
  metadata={},  # metadata for the batch
  extra_body={
    "input_blob": "<blob_url>",
    "output_folder": {
      "url": "<output_blob_folder>"  # both error file and output file will be saved in this folder
    }
  }
)

print(start_batch_response)
```

### List Batch Jobs

```python Python theme={"system"}
from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

batches = portkey.batches.list()

print(batches)
```

```javascript Typescript theme={"system"}
import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const listBatches = async () => {
  const batches = await portkey.batches.list();

  console.log(batches);
}

await listBatches();

```

```bash curl theme={"system"}
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
```

```javascript OpenAI NodeJS theme={"system"}
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const listBatches = async () => {
  const batches = await openai.batches.list();

  console.log(batches);
}

await listBatches();
```

```python OpenAI Python theme={"system"}
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

batches = openai.batches.list()

print(batches)
```

### Get Batch Job Details

```python Python theme={"system"}
from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

batch = portkey.batches.retrieve(batch_id="batch_id")

print(batch)
```

```javascript Typescript theme={"system"}
import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const getBatch = async () => {
  const batch = await portkey.batches.retrieve(batch_id="batch_id");

  console.log(batch);
}

await getBatch();

```

```bash curl theme={"system"}
curl --location 'https://api.portkey.ai/v1/batches/<batch_id>' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
```

```javascript OpenAI NodeJS theme={"system"}
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const getBatch = async () => {
  const batch = await openai.batches.retrieve(batch_id="batch_id");

  console.log(batch);
}

await getBatch();
```

```python OpenAI Python theme={"system"}
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

batch = openai.batches.retrieve(batch_id="batch_id")

print(batch)
```

### Get Batch Output

```bash curl theme={"system"}
curl --location 'https://api.portkey.ai/v1/batches/<batch_id>/output' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
```

### List Batch Jobs

```python Python theme={"system"}
from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

batches = portkey.batches.list()

print(batches)
```

```javascript Typescript theme={"system"}
import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const listBatchingJobs = async () => {
  const batching_jobs = await portkey.batches.list();

  console.log(batching_jobs);
}

await listBatchingJobs();

```

```bash curl theme={"system"}
curl --location 'https://api.portkey.ai/v1/batches' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
```

```javascript OpenAI NodeJS theme={"system"}
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const listBatchingJobs = async () => {
  const batching_jobs = await openai.batches.list();

  console.log(batching_jobs);
}

await listBatchingJobs();
```

```python OpenAI Python theme={"system"}
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

batching_jobs = openai.batches.list()

print(batching_jobs)
```

### Cancel Batch Job

```python Python theme={"system"}
from portkey_ai import Portkey

# Initialize the Portkey client
portkey = Portkey(
    api_key="PORTKEY_API_KEY",  # Replace with your Portkey API key
    provider="@PROVIDER"   
)

cancel_batch_response = portkey.batches.cancel(batch_id="batch_id")

print(cancel_batch_response)
```

```javascript Typescript theme={"system"}
import { Portkey } from 'portkey-ai';

// Initialize the Portkey client
const portkey = new Portkey({
    apiKey: "PORTKEY_API_KEY",  // Replace with your Portkey API key
    provider:"@PROVIDER"   
});

const cancelBatch = async () => {
  const cancel_batch_response = await portkey.batches.cancel(batch_id="batch_id");

  console.log(cancel_batch_response);
}

await cancelBatch();

```

```bash curl theme={"system"}
curl --request POST --location 'https://api.portkey.ai/v1/batches/<batch_id>/cancel' \
--header 'x-portkey-api-key: <portkey_api_key>' \
--header 'x-portkey-provider: @provider'
```

```javascript OpenAI NodeJS theme={"system"}
import OpenAI from 'openai'; // We're using the v4 SDK
import { PORTKEY_GATEWAY_URL, createHeaders } from 'portkey-ai'

const openai = new OpenAI({
  apiKey: 'OPENAI_API_KEY', // defaults to process.env["OPENAI_API_KEY"],
  baseURL: PORTKEY_GATEWAY_URL,
  defaultHeaders: createHeaders({
    provider: "openai",
    apiKey: "PORTKEY_API_KEY" // defaults to process.env["PORTKEY_API_KEY"]
  })
});

const cancelBatch = async () => {
  const cancel_batch_response = await openai.batches.cancel(batch_id="batch_id");

  console.log(cancel_batch_response);
}

await cancelBatch();
```

```python OpenAI Python theme={"system"}
from openai import OpenAI
from portkey_ai import PORTKEY_GATEWAY_URL, createHeaders

openai = OpenAI(
    api_key='OPENAI_API_KEY',
    base_url=PORTKEY_GATEWAY_URL,
    default_headers=createHeaders(
        provider="openai",
        api_key="PORTKEY_API_KEY"
    )
)

cancel_batch_response = openai.batches.cancel(batch_id="batch_id")

print(cancel_batch_response)
```

***

> [card] [Portkey is now PRISMA AIRS AI Gateway. See it in action.](https://www.paloaltonetworks.in/ai-security/ai-gateway?utm_source=portkey\&utm_medium=referral\&utm_campaign=prisma_airs\&utm_content=docs_nav#contact)
>
> Contact Us


## Related topics

- [Batches](https://docs.portkey.ai/docs/integrations/llms/anthropic/batches.md)


This documentation is built and hosted on [Mintlify](https://mintlify.com), a developer documentation platform.