{"info":{"title":"Hugging Face Endpoints API","version":"1.0.0"},"openapi":"3.1.0","paths":{"/catalog/list":{"get":{"operationId":"listCatalogItems","responses":{"200":{"description":"Matching catalog items.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"items":{"type":"array","items":{"type":"object","properties":{"modelName":{"type":"string"},"createdAt":{"type":"string","format":"date-time","pattern":"^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z))$"},"authorName":{"type":"string"},"license":{"type":"string"},"repoId":{"type":"string"},"task":{"type":"string"},"recipes":{"type":"array","items":{"type":"object","properties":{"accelerator":{"type":"string","enum":["cpu","gpu","neuron"]},"engineType":{"type":"string","enum":["custom","llamacpp","sglang","tei","tgi","tgiNeuron","vllm","vllmNeuron","vllmOmni"]},"ggufFile":{"type":"string"},"publicId":{"type":"string","description":"Pass as recipe_public_id to get_recommended_config to target this exact recipe."},"revision":{"description":"The Hugging Face Hub model commit SHA or branch this recipe is pinned to, if any.","type":"string"}},"required":["accelerator","engineType","publicId"],"additionalProperties":false}}},"required":["modelName","createdAt","authorName","license","repoId","task","recipes"],"additionalProperties":false}}},"required":["items"],"additionalProperties":false}}}},"400":{"description":"Invalid query parameters.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"details":{},"error":{"type":"string"}},"required":["details","error"],"additionalProperties":false}}}},"500":{"description":"Server error while loading catalog items.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}}},"summary":"List catalog items","parameters":[{"in":"query","name":"accelerator","required":false,"schema":{"type":"string","enum":["cpu","gpu","neuron"]}},{"in":"query","name":"engine","required":false,"schema":{"type":"string","enum":["llamacpp","tei","vllm","sglang"]}},{"in":"query","name":"license","required":false,"schema":{"type":"string"}},{"in":"query","name":"limit","required":false,"schema":{"type":"integer","exclusiveMinimum":0,"maximum":9007199254740991}},{"in":"query","name":"search","required":false,"schema":{"type":"string"}},{"in":"query","name":"task","required":false,"schema":{"type":"string","enum":["any-to-any","audio-classification","automatic-speech-recognition","custom","feature-extraction","fill-mask","image-classification","image-segmentation","image-text-to-text","object-detection","question-answering","sentence-similarity","summarization","table-question-answering","text-classification","text-generation","text-ranking","text-to-image","token-classification","translation","zero-shot-classification"]}}]}},"/catalog/model/{author}/{model}/deploy":{"post":{"operationId":"deployCatalogModel","responses":{"200":{"description":"The created endpoint.","content":{"application/json":{"schema":{"properties":{"endpoint":{"type":"object","description":"Endpoint","required":["name","type","provider","compute","model"],"properties":{"cacheHttpResponses":{"type":"boolean"},"compute":{"type":"object","description":"Compute resources allocated for each endpoint replica","required":["accelerator","instanceType","instanceSize","scaling"],"properties":{"accelerator":{"type":"string","enum":["cpu","gpu","neuron","zero_gpu"],"description":"Accelerator type"},"id":{"type":["string","null"],"description":"Only informational, ignored at specification"},"instanceSize":{"type":"string","example":"x1"},"instanceType":{"type":"string","description":"Compute instance type","example":"intel-spr"},"scaling":{"type":"object","description":"Scaling","required":["minReplica","maxReplica"],"properties":{"maxReplica":{"type":"integer","format":"int32","description":"Maximum number of replicas","example":4,"minimum":0},"measure":{"oneOf":[{"type":"null"},{"oneOf":[{"type":"object","description":"Scale when hardware utilization exceeds threshold (0-100%)","required":["hardwareUsage"],"properties":{"hardwareUsage":{"type":["number","null"],"format":"double","description":"Scale when hardware utilization exceeds threshold (0-100%)"}},"example":{"hardwareUsage":80}},{"type":"object","description":"Scale when pending requests exceed threshold","required":["pendingRequests"],"properties":{"pendingRequests":{"type":["number","null"],"format":"double","description":"Scale when pending requests exceed threshold"}},"example":{"pendingRequests":10}}],"description":"Scaling metric and threshold"}]},"metric":{"oneOf":[{"type":"null"},{"type":"string","description":"Deprecated: use `measure` instead","enum":["hardwareUsage","pendingRequests"],"deprecated":true}]},"minReplica":{"type":"integer","format":"int32","description":"Minimum number of replicas (set to 0 to enable scale-to-zero)","example":1,"minimum":0},"scaleToZeroTimeout":{"type":["integer","null"],"format":"int32","description":"Minutes of inactivity before scaling to zero (default: 15, requires minReplica=0)","example":15,"minimum":0},"threshold":{"type":["number","null"],"format":"double","description":"Deprecated: threshold is now part of `measure`","deprecated":true}}}}},"customRouter":{"oneOf":[{"type":"null"},{"type":"object","description":"Use a custom-router sidecar to load balance requests between replicas","properties":{"env":{"type":"object","description":"Additional environment variables to set on the custom-router sidecar","additionalProperties":{"type":"string"},"propertyNames":{"type":"string"}},"grpcPort":{"type":["integer","null"],"format":"int32","description":"Port of the sidecar's Envoy `ExternalProcessor` gRPC service, used with `protocol: extProc`.\nDefaults to 9002.","minimum":0},"port":{"type":["integer","null"],"format":"int32","description":"Port the custom-router sidecar listens on. Defaults to 3000.","minimum":0},"protocol":{"type":"string","enum":["http","extProc"],"description":"How the proxy hands requests to the sidecar. Defaults to `http`."},"tag":{"type":["string","null"],"description":"Custom-router sidecar image tag. When null or absent, the custom router is removed."}}}]},"experimentalFeatures":{"type":"object","description":"Experimental features, may be accessible on demand","properties":{"cacheHttpResponses":{"type":"boolean","description":"Enable caching of successful HTTP responses. When enabled, responses are\ncached with a 60min TTL (reset on each cache hit) and served instantly\non identical requests. With 100% cache hit rates, endpoints can\nautomatically scale to zero while still serving cached responses,\noptimizing both latency and cost. Cache hits are indicated by the\n'x-cache: HIT' response header, and requests can bypass the cache using\n'x-use-cache: false'.\nCurrently only available for HuggingFace Admins."},"loadBalancer":{"oneOf":[{"type":"null"},{"type":"string","description":"Load balancer strategy for distributing requests across replicas.\nDefaults to `random`. Has no effect when `custom_router` or `mcp_lb` is set.","enum":["random","sticky","least_connections"]}]},"mcpLb":{"oneOf":[{"type":"null"},{"type":"object","properties":{"stateful":{"type":"boolean","description":"If true, we forward the request following the mcp session id"}},"description":"Use MCP to load balance requests between replicas"}]},"skipRepoDownload":{"type":["boolean","null"],"description":"Skip model repository download on initialization"}}},"model":{"type":"object","description":"Model served by the endpoint","required":["repository","framework","image"],"properties":{"args":{"type":["array","null"],"items":{"type":"string"},"description":"Arguments passed to the container command"},"command":{"type":["array","null"],"items":{"type":"string"},"description":"Override container entrypoint command"},"env":{"type":"object","description":"Environment variables","additionalProperties":{"type":"string"},"propertyNames":{"type":"string"},"example":{"MAX_CONCURRENT_REQUESTS":"128"}},"framework":{"type":"string","description":"ML framework","enum":["custom","pytorch","llamacpp"]},"fromCatalog":{"type":["boolean","null"],"description":"Indicates the endpoint was created from a catalog template"},"image":{"oneOf":[{"type":"object","required":["huggingface"],"properties":{"huggingface":{"type":"object","description":"Model served by a Hugging Face container"}}},{"type":"object","required":["huggingfaceNeuron"],"properties":{"huggingfaceNeuron":{"type":"object","description":"Model served by a Hugging Face container specific to Neuronx accelerators","properties":{"batchSize":{"type":["integer","null"],"format":"int32","description":"Batch size","minimum":0},"neuronCache":{"type":"string","description":"Optimum neuron cache"},"sequenceLength":{"type":["integer","null"],"format":"int32","description":"Sequence length","minimum":0}}}}},{"type":"object","required":["tgi"],"properties":{"tgi":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"$ref":"#/components/schemas/BaseTgiContainer"},{"type":"object","properties":{"disableCustomKernels":{"type":"boolean","description":"Whether or not custom cuda kernels should be used when available","default":true,"example":true},"quantize":{"oneOf":[{"type":"null"},{"type":"string","enum":["awq","bitsandbytes","eetq","gptq"],"description":"Model quantize"}]}}}],"description":"Model served by Text Generation Inference"}}},{"type":"object","required":["tgiNeuron"],"properties":{"tgiNeuron":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"$ref":"#/components/schemas/BaseTgiContainer"},{"type":"object","properties":{"hfAutoCastType":{"oneOf":[{"type":"null"},{"type":"string","enum":["bf16","fp16"],"description":"Auto cast type"}]},"hfNumCores":{"type":["integer","null"],"format":"int32","description":"Number of cores","minimum":0}}}],"description":"Model served by a Text Generation Inference specific to Neuronx accelerators"}}},{"type":"object","required":["tei"],"properties":{"tei":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"maxBatchTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens before moving requests to the next batch","example":2048,"minimum":0},"maxConcurrentRequests":{"type":["integer","null"],"format":"int32","description":"Maximum number of concurrent requests","example":100,"minimum":0},"pooling":{"oneOf":[{"type":"null"},{"type":"string","enum":["mean","cls"],"description":"Pooling parameter"}]}}}],"description":"Model served by Text Embedding Inference"}}},{"type":"object","required":["llamacpp"],"properties":{"llamacpp":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","required":["modelPath"],"properties":{"ctxSize":{"type":["integer","null"],"format":"int32","description":"Size of the KV cache. Left unset or 0, `--fit` sizes the context to the accelerator's free memory","example":8192,"minimum":0},"mmprojModelPath":{"type":["string","null"],"description":"Path to the .gguf vision model file","example":"mmproj-my_model.Q8_0.gguf"},"mode":{"oneOf":[{"type":"null"},{"type":"string","enum":["embeddings","reranking"],"description":"Model mode (if unspecified defaults to generation mode)"}]},"modelPath":{"type":"string","description":"Path to the .gguf file to be loaded","example":"my_model.Q8_0.gguf"},"nGpuLayers":{"type":["integer","null"],"format":"int32","description":"Number of layers to offload to the GPU. Left unset, llama.cpp picks the value itself\n(and `--fit` can size the context to the accelerator's free memory)","example":9999,"minimum":0},"nParallel":{"type":["integer","null"],"format":"int32","description":"Maximum number of requests in parallel. Left unset, llama.cpp picks the value itself","example":1,"minimum":0},"pooling":{"oneOf":[{"type":"null"},{"type":"string","enum":["mean","cls","last","rank"],"description":"Model pooling"}]},"specModelPath":{"type":["string","null"],"description":"Path to the .gguf draft model file, used for speculative decoding","example":"mtp-my_model.Q8_0.gguf"},"threadsHttp":{"type":["integer","null"],"format":"int32","description":"Number of threads for the HTTP server. Left unset, llama.cpp picks the value itself","example":64,"minimum":0},"variant":{"type":["string","null"],"description":"Pattern of .gguf files to load","example":"*"}}}],"description":"Model served by llama.cpp. User-defined args full list can be found at https://github.com/ggerganov/llama.cpp/tree/master/examples/server/README.md"}}},{"type":"object","required":["vLLM"],"properties":{"vLLM":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"dataParallelSize":{"type":["integer","null"],"format":"int32","description":"Number of data parallel groups","example":2,"minimum":0},"kvCacheDtype":{"oneOf":[{"type":"null"},{"type":"string","description":"Data type for kv cache","enum":["fp8","fp8-e4-m3","fp8-e5-m2"]}]},"maxNumBatchedTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens to be processed in a single iteration","example":256,"minimum":0},"maxNumSeqs":{"type":["integer","null"],"format":"int32","description":"Maximum number of sequences to be processed in a single iteration.","example":8096,"minimum":0},"tensorParallelSize":{"type":["integer","null"],"format":"int32","description":"Number of tensor parallel groups","example":8,"minimum":0}}}],"description":"Model served by vLLM"}}},{"type":"object","required":["vLLMOmni"],"properties":{"vLLMOmni":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"deployConfig":{"type":["string","null"],"description":"Path to a vLLM-Omni deployment configuration inside the container","example":"/repository/deploy.yaml"},"maxNumSeqs":{"type":["integer","null"],"format":"int32","description":"Maximum number of requests processed concurrently by each stage","example":16,"minimum":0}}}],"description":"Model served by vLLM-Omni"}}},{"type":"object","required":["vLLMNeuron"],"properties":{"vLLMNeuron":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"}],"description":"Model served by vLLM on Neuronx accelerators"}}},{"type":"object","required":["sGLang"],"properties":{"sGLang":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"chunkedPrefillSize":{"type":["integer","null"],"format":"int32","description":"The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked\nprefill","example":4096,"minimum":0},"kvCacheDtype":{"oneOf":[{"type":"null"},{"type":"string","description":"Data type for kv cache","enum":["fp8-e4-m3","fp8-e5-m2"]}]},"maxPrefillTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens that can be processed in a single prefill batch","example":8192,"minimum":0},"maxRunningRequests":{"type":["integer","null"],"format":"int32","description":"Sets the maximum number of concurrent requests","example":64,"minimum":0},"tensorParallelSize":{"type":["integer","null"],"format":"int32","description":"Number of tensor parallel groups","example":2,"minimum":0}}}],"description":"Model served by"}}},{"type":"object","required":["hfServe"],"properties":{"hfServe":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"}],"description":"Model served by the HF-serve framework"}}},{"type":"object","required":["custom"],"properties":{"custom":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"credentials":{"oneOf":[{"type":"null"},{"type":"object","description":"Credentials to download the image","required":["username","password"],"properties":{"password":{"oneOf":[{"type":"null"},{"type":"string"}]},"username":{"type":"string","description":"Registry username"}}}]}}}],"description":"Model served by a custom container"}}}],"description":"Container image configuration"},"repository":{"type":"string","description":"HuggingFace model repository (e.g., \"meta-llama/Llama-2-7b-hf\")","example":"meta-llama/Llama-2-7b-hf"},"revision":{"type":["string","null"],"description":"Git commit SHA or branch name (defaults to latest main)","example":"main"},"secrets":{"type":"object","description":"Secret environment variables (values are encrypted at rest)","additionalProperties":{"type":["string","null"]},"propertyNames":{"type":"string"},"example":{"HF_TOKEN":"hf_xxx"}},"task":{"type":"string","description":"Model task type for inference","enum":["custom","any-to-any","audio-classification","automatic-speech-recognition","feature-extraction","fill-mask","image-classification","image-segmentation","image-text-to-text","object-detection","question-answering","sentence-embeddings","sentence-ranking","sentence-similarity","summarization","table-question-answering","text-classification","text-generation","text-ranking","text-to-image","token-classification","translation","zero-shot-classification"]}}},"name":{"type":"string","description":"Endpoint name. Must only contains lowercase alphanumeric characters or '-' and have a length of 32 characters\nmaximum","example":"my-endpoint"},"network":{"oneOf":[{"type":"null"},{"type":"object","description":"Network specification","properties":{"cidrBlock":{"type":["boolean","null"],"description":"Enable CIDR-based access restriction"},"cidrList":{"type":["array","null"],"items":{"type":"string"},"description":"List of allowed CIDR blocks (e.g., [\"10.0.0.0/8\", \"192.168.1.0/24\"])","example":["10.0.0.0/8"]}}}]},"notifications":{"type":"object","description":"Configure endpoint events notifications","required":["email","push"],"properties":{"email":{"type":"boolean","description":"Enable email notifications"},"push":{"type":"boolean","description":"Enable front push notifications"}}},"privateService":{"oneOf":[{"type":"null"},{"type":"object","description":"Endpoint private service specification (for private endpoints only)","required":["accountId"],"properties":{"accountId":{"type":"string","description":"AWS account ID allowed to connect via PrivateLink","example":"123456789012"},"shared":{"type":"boolean","description":"Share the PrivateLink service across all endpoints for this owner (reduces costs)"}}}]},"provider":{"type":"object","description":"Provider on which the endpoint is deployed","required":["vendor","region"],"properties":{"region":{"type":"string","description":"Provider region","example":"us-east-1"},"vendor":{"type":"string","description":"Provider vendor","example":"aws"}}},"route":{"oneOf":[{"type":"null"},{"type":"object","description":"Route spec for multipath routing behind a custom domain","required":["domain","path"],"properties":{"domain":{"type":"string","description":"Custom domain name","example":"api.example.com"},"path":{"type":"string","description":"Path prefix for routing","example":"/v1/models/gpt2"}}}]},"tags":{"type":"array","items":{"type":"string"},"description":"Endpoint tags"},"type":{"type":"string","description":"Endpoint type","enum":["public","authenticated","private"]}}}},"required":["endpoint"],"type":"object"}}}},"400":{"description":"Invalid request body.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"},"issues":{"type":"array","items":{}}},"required":["error","issues"],"additionalProperties":false}}}},"401":{"description":"Missing or invalid bearer token.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}},"404":{"description":"Catalog item, recipe, or model not found.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}},"409":{"description":"No compatible compute found for this model.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}},"500":{"description":"Server error.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}}},"summary":"Deploy a catalog model as an Inference Endpoint","parameters":[{"in":"path","name":"author","required":true,"schema":{"type":"string"}},{"in":"path","name":"model","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"accelerator":{"type":"string","enum":["cpu","gpu","neuron"]},"config":{"type":"object","properties":{"name":{"type":"string"},"privateService":{"type":"object","properties":{"accountId":{"type":"string"},"shared":{"type":"boolean"}},"required":["accountId"],"additionalProperties":false},"scaling":{"type":"object","properties":{"maxReplica":{"type":"number"},"minReplica":{"type":"number"},"scaleToZeroTimeout":{"type":"number"}},"additionalProperties":false},"type":{"type":"string","enum":["private","public","authenticated"]}},"additionalProperties":false},"ggufFile":{"type":"string"},"namespace":{"type":"string"}},"required":["namespace"],"additionalProperties":false}}},"required":true}}},"/catalog/recipe/{recipePublicId}/deploy":{"post":{"operationId":"deployCatalogRecipe","responses":{"200":{"description":"The created endpoint.","content":{"application/json":{"schema":{"properties":{"endpoint":{"type":"object","description":"Endpoint","required":["name","type","provider","compute","model"],"properties":{"cacheHttpResponses":{"type":"boolean"},"compute":{"type":"object","description":"Compute resources allocated for each endpoint replica","required":["accelerator","instanceType","instanceSize","scaling"],"properties":{"accelerator":{"type":"string","enum":["cpu","gpu","neuron","zero_gpu"],"description":"Accelerator type"},"id":{"type":["string","null"],"description":"Only informational, ignored at specification"},"instanceSize":{"type":"string","example":"x1"},"instanceType":{"type":"string","description":"Compute instance type","example":"intel-spr"},"scaling":{"type":"object","description":"Scaling","required":["minReplica","maxReplica"],"properties":{"maxReplica":{"type":"integer","format":"int32","description":"Maximum number of replicas","example":4,"minimum":0},"measure":{"oneOf":[{"type":"null"},{"oneOf":[{"type":"object","description":"Scale when hardware utilization exceeds threshold (0-100%)","required":["hardwareUsage"],"properties":{"hardwareUsage":{"type":["number","null"],"format":"double","description":"Scale when hardware utilization exceeds threshold (0-100%)"}},"example":{"hardwareUsage":80}},{"type":"object","description":"Scale when pending requests exceed threshold","required":["pendingRequests"],"properties":{"pendingRequests":{"type":["number","null"],"format":"double","description":"Scale when pending requests exceed threshold"}},"example":{"pendingRequests":10}}],"description":"Scaling metric and threshold"}]},"metric":{"oneOf":[{"type":"null"},{"type":"string","description":"Deprecated: use `measure` instead","enum":["hardwareUsage","pendingRequests"],"deprecated":true}]},"minReplica":{"type":"integer","format":"int32","description":"Minimum number of replicas (set to 0 to enable scale-to-zero)","example":1,"minimum":0},"scaleToZeroTimeout":{"type":["integer","null"],"format":"int32","description":"Minutes of inactivity before scaling to zero (default: 15, requires minReplica=0)","example":15,"minimum":0},"threshold":{"type":["number","null"],"format":"double","description":"Deprecated: threshold is now part of `measure`","deprecated":true}}}}},"customRouter":{"oneOf":[{"type":"null"},{"type":"object","description":"Use a custom-router sidecar to load balance requests between replicas","properties":{"env":{"type":"object","description":"Additional environment variables to set on the custom-router sidecar","additionalProperties":{"type":"string"},"propertyNames":{"type":"string"}},"grpcPort":{"type":["integer","null"],"format":"int32","description":"Port of the sidecar's Envoy `ExternalProcessor` gRPC service, used with `protocol: extProc`.\nDefaults to 9002.","minimum":0},"port":{"type":["integer","null"],"format":"int32","description":"Port the custom-router sidecar listens on. Defaults to 3000.","minimum":0},"protocol":{"type":"string","enum":["http","extProc"],"description":"How the proxy hands requests to the sidecar. Defaults to `http`."},"tag":{"type":["string","null"],"description":"Custom-router sidecar image tag. When null or absent, the custom router is removed."}}}]},"experimentalFeatures":{"type":"object","description":"Experimental features, may be accessible on demand","properties":{"cacheHttpResponses":{"type":"boolean","description":"Enable caching of successful HTTP responses. When enabled, responses are\ncached with a 60min TTL (reset on each cache hit) and served instantly\non identical requests. With 100% cache hit rates, endpoints can\nautomatically scale to zero while still serving cached responses,\noptimizing both latency and cost. Cache hits are indicated by the\n'x-cache: HIT' response header, and requests can bypass the cache using\n'x-use-cache: false'.\nCurrently only available for HuggingFace Admins."},"loadBalancer":{"oneOf":[{"type":"null"},{"type":"string","description":"Load balancer strategy for distributing requests across replicas.\nDefaults to `random`. Has no effect when `custom_router` or `mcp_lb` is set.","enum":["random","sticky","least_connections"]}]},"mcpLb":{"oneOf":[{"type":"null"},{"type":"object","properties":{"stateful":{"type":"boolean","description":"If true, we forward the request following the mcp session id"}},"description":"Use MCP to load balance requests between replicas"}]},"skipRepoDownload":{"type":["boolean","null"],"description":"Skip model repository download on initialization"}}},"model":{"type":"object","description":"Model served by the endpoint","required":["repository","framework","image"],"properties":{"args":{"type":["array","null"],"items":{"type":"string"},"description":"Arguments passed to the container command"},"command":{"type":["array","null"],"items":{"type":"string"},"description":"Override container entrypoint command"},"env":{"type":"object","description":"Environment variables","additionalProperties":{"type":"string"},"propertyNames":{"type":"string"},"example":{"MAX_CONCURRENT_REQUESTS":"128"}},"framework":{"type":"string","description":"ML framework","enum":["custom","pytorch","llamacpp"]},"fromCatalog":{"type":["boolean","null"],"description":"Indicates the endpoint was created from a catalog template"},"image":{"oneOf":[{"type":"object","required":["huggingface"],"properties":{"huggingface":{"type":"object","description":"Model served by a Hugging Face container"}}},{"type":"object","required":["huggingfaceNeuron"],"properties":{"huggingfaceNeuron":{"type":"object","description":"Model served by a Hugging Face container specific to Neuronx accelerators","properties":{"batchSize":{"type":["integer","null"],"format":"int32","description":"Batch size","minimum":0},"neuronCache":{"type":"string","description":"Optimum neuron cache"},"sequenceLength":{"type":["integer","null"],"format":"int32","description":"Sequence length","minimum":0}}}}},{"type":"object","required":["tgi"],"properties":{"tgi":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"$ref":"#/components/schemas/BaseTgiContainer"},{"type":"object","properties":{"disableCustomKernels":{"type":"boolean","description":"Whether or not custom cuda kernels should be used when available","default":true,"example":true},"quantize":{"oneOf":[{"type":"null"},{"type":"string","enum":["awq","bitsandbytes","eetq","gptq"],"description":"Model quantize"}]}}}],"description":"Model served by Text Generation Inference"}}},{"type":"object","required":["tgiNeuron"],"properties":{"tgiNeuron":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"$ref":"#/components/schemas/BaseTgiContainer"},{"type":"object","properties":{"hfAutoCastType":{"oneOf":[{"type":"null"},{"type":"string","enum":["bf16","fp16"],"description":"Auto cast type"}]},"hfNumCores":{"type":["integer","null"],"format":"int32","description":"Number of cores","minimum":0}}}],"description":"Model served by a Text Generation Inference specific to Neuronx accelerators"}}},{"type":"object","required":["tei"],"properties":{"tei":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"maxBatchTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens before moving requests to the next batch","example":2048,"minimum":0},"maxConcurrentRequests":{"type":["integer","null"],"format":"int32","description":"Maximum number of concurrent requests","example":100,"minimum":0},"pooling":{"oneOf":[{"type":"null"},{"type":"string","enum":["mean","cls"],"description":"Pooling parameter"}]}}}],"description":"Model served by Text Embedding Inference"}}},{"type":"object","required":["llamacpp"],"properties":{"llamacpp":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","required":["modelPath"],"properties":{"ctxSize":{"type":["integer","null"],"format":"int32","description":"Size of the KV cache. Left unset or 0, `--fit` sizes the context to the accelerator's free memory","example":8192,"minimum":0},"mmprojModelPath":{"type":["string","null"],"description":"Path to the .gguf vision model file","example":"mmproj-my_model.Q8_0.gguf"},"mode":{"oneOf":[{"type":"null"},{"type":"string","enum":["embeddings","reranking"],"description":"Model mode (if unspecified defaults to generation mode)"}]},"modelPath":{"type":"string","description":"Path to the .gguf file to be loaded","example":"my_model.Q8_0.gguf"},"nGpuLayers":{"type":["integer","null"],"format":"int32","description":"Number of layers to offload to the GPU. Left unset, llama.cpp picks the value itself\n(and `--fit` can size the context to the accelerator's free memory)","example":9999,"minimum":0},"nParallel":{"type":["integer","null"],"format":"int32","description":"Maximum number of requests in parallel. Left unset, llama.cpp picks the value itself","example":1,"minimum":0},"pooling":{"oneOf":[{"type":"null"},{"type":"string","enum":["mean","cls","last","rank"],"description":"Model pooling"}]},"specModelPath":{"type":["string","null"],"description":"Path to the .gguf draft model file, used for speculative decoding","example":"mtp-my_model.Q8_0.gguf"},"threadsHttp":{"type":["integer","null"],"format":"int32","description":"Number of threads for the HTTP server. Left unset, llama.cpp picks the value itself","example":64,"minimum":0},"variant":{"type":["string","null"],"description":"Pattern of .gguf files to load","example":"*"}}}],"description":"Model served by llama.cpp. User-defined args full list can be found at https://github.com/ggerganov/llama.cpp/tree/master/examples/server/README.md"}}},{"type":"object","required":["vLLM"],"properties":{"vLLM":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"dataParallelSize":{"type":["integer","null"],"format":"int32","description":"Number of data parallel groups","example":2,"minimum":0},"kvCacheDtype":{"oneOf":[{"type":"null"},{"type":"string","description":"Data type for kv cache","enum":["fp8","fp8-e4-m3","fp8-e5-m2"]}]},"maxNumBatchedTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens to be processed in a single iteration","example":256,"minimum":0},"maxNumSeqs":{"type":["integer","null"],"format":"int32","description":"Maximum number of sequences to be processed in a single iteration.","example":8096,"minimum":0},"tensorParallelSize":{"type":["integer","null"],"format":"int32","description":"Number of tensor parallel groups","example":8,"minimum":0}}}],"description":"Model served by vLLM"}}},{"type":"object","required":["vLLMOmni"],"properties":{"vLLMOmni":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"deployConfig":{"type":["string","null"],"description":"Path to a vLLM-Omni deployment configuration inside the container","example":"/repository/deploy.yaml"},"maxNumSeqs":{"type":["integer","null"],"format":"int32","description":"Maximum number of requests processed concurrently by each stage","example":16,"minimum":0}}}],"description":"Model served by vLLM-Omni"}}},{"type":"object","required":["vLLMNeuron"],"properties":{"vLLMNeuron":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"}],"description":"Model served by vLLM on Neuronx accelerators"}}},{"type":"object","required":["sGLang"],"properties":{"sGLang":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"chunkedPrefillSize":{"type":["integer","null"],"format":"int32","description":"The maximum number of tokens in a chunk for the chunked prefill. Setting this to -1 means disabling chunked\nprefill","example":4096,"minimum":0},"kvCacheDtype":{"oneOf":[{"type":"null"},{"type":"string","description":"Data type for kv cache","enum":["fp8-e4-m3","fp8-e5-m2"]}]},"maxPrefillTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens that can be processed in a single prefill batch","example":8192,"minimum":0},"maxRunningRequests":{"type":["integer","null"],"format":"int32","description":"Sets the maximum number of concurrent requests","example":64,"minimum":0},"tensorParallelSize":{"type":["integer","null"],"format":"int32","description":"Number of tensor parallel groups","example":2,"minimum":0}}}],"description":"Model served by"}}},{"type":"object","required":["hfServe"],"properties":{"hfServe":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"}],"description":"Model served by the HF-serve framework"}}},{"type":"object","required":["custom"],"properties":{"custom":{"allOf":[{"$ref":"#/components/schemas/BaseContainer"},{"type":"object","properties":{"credentials":{"oneOf":[{"type":"null"},{"type":"object","description":"Credentials to download the image","required":["username","password"],"properties":{"password":{"oneOf":[{"type":"null"},{"type":"string"}]},"username":{"type":"string","description":"Registry username"}}}]}}}],"description":"Model served by a custom container"}}}],"description":"Container image configuration"},"repository":{"type":"string","description":"HuggingFace model repository (e.g., \"meta-llama/Llama-2-7b-hf\")","example":"meta-llama/Llama-2-7b-hf"},"revision":{"type":["string","null"],"description":"Git commit SHA or branch name (defaults to latest main)","example":"main"},"secrets":{"type":"object","description":"Secret environment variables (values are encrypted at rest)","additionalProperties":{"type":["string","null"]},"propertyNames":{"type":"string"},"example":{"HF_TOKEN":"hf_xxx"}},"task":{"type":"string","description":"Model task type for inference","enum":["custom","any-to-any","audio-classification","automatic-speech-recognition","feature-extraction","fill-mask","image-classification","image-segmentation","image-text-to-text","object-detection","question-answering","sentence-embeddings","sentence-ranking","sentence-similarity","summarization","table-question-answering","text-classification","text-generation","text-ranking","text-to-image","token-classification","translation","zero-shot-classification"]}}},"name":{"type":"string","description":"Endpoint name. Must only contains lowercase alphanumeric characters or '-' and have a length of 32 characters\nmaximum","example":"my-endpoint"},"network":{"oneOf":[{"type":"null"},{"type":"object","description":"Network specification","properties":{"cidrBlock":{"type":["boolean","null"],"description":"Enable CIDR-based access restriction"},"cidrList":{"type":["array","null"],"items":{"type":"string"},"description":"List of allowed CIDR blocks (e.g., [\"10.0.0.0/8\", \"192.168.1.0/24\"])","example":["10.0.0.0/8"]}}}]},"notifications":{"type":"object","description":"Configure endpoint events notifications","required":["email","push"],"properties":{"email":{"type":"boolean","description":"Enable email notifications"},"push":{"type":"boolean","description":"Enable front push notifications"}}},"privateService":{"oneOf":[{"type":"null"},{"type":"object","description":"Endpoint private service specification (for private endpoints only)","required":["accountId"],"properties":{"accountId":{"type":"string","description":"AWS account ID allowed to connect via PrivateLink","example":"123456789012"},"shared":{"type":"boolean","description":"Share the PrivateLink service across all endpoints for this owner (reduces costs)"}}}]},"provider":{"type":"object","description":"Provider on which the endpoint is deployed","required":["vendor","region"],"properties":{"region":{"type":"string","description":"Provider region","example":"us-east-1"},"vendor":{"type":"string","description":"Provider vendor","example":"aws"}}},"route":{"oneOf":[{"type":"null"},{"type":"object","description":"Route spec for multipath routing behind a custom domain","required":["domain","path"],"properties":{"domain":{"type":"string","description":"Custom domain name","example":"api.example.com"},"path":{"type":"string","description":"Path prefix for routing","example":"/v1/models/gpt2"}}}]},"tags":{"type":"array","items":{"type":"string"},"description":"Endpoint tags"},"type":{"type":"string","description":"Endpoint type","enum":["public","authenticated","private"]}}}},"required":["endpoint"],"type":"object"}}}},"400":{"description":"Invalid request body.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"},"issues":{"type":"array","items":{}}},"required":["error","issues"],"additionalProperties":false}}}},"401":{"description":"Missing or invalid bearer token.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}},"404":{"description":"Catalog item, recipe, or model not found.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}},"409":{"description":"No compatible compute found for this model.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}},"500":{"description":"Server error.","content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"error":{"type":"string"}},"required":["error"],"additionalProperties":false}}}}},"summary":"Deploy a specific catalog recipe as an Inference Endpoint","parameters":[{"in":"path","name":"recipePublicId","required":true,"schema":{"type":"string"}}],"requestBody":{"content":{"application/json":{"schema":{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"config":{"type":"object","properties":{"name":{"type":"string"},"privateService":{"type":"object","properties":{"accountId":{"type":"string"},"shared":{"type":"boolean"}},"required":["accountId"],"additionalProperties":false},"scaling":{"type":"object","properties":{"maxReplica":{"type":"number"},"minReplica":{"type":"number"},"scaleToZeroTimeout":{"type":"number"}},"additionalProperties":false},"type":{"type":"string","enum":["private","public","authenticated"]}},"additionalProperties":false},"namespace":{"type":"string"}},"required":["namespace"],"additionalProperties":false}}},"required":true}}}},"security":[],"servers":[{"url":"/api/v1"}],"components":{"schemas":{"BaseContainer":{"type":"object","required":["url"],"properties":{"healthRoute":{"type":["string","null"],"description":"Container health route","example":"/health"},"port":{"type":["integer","null"],"format":"int32","description":"Endpoint API port. If not specified, uses the container's default port.","example":80,"minimum":0},"url":{"type":"string","description":"URL for the container","example":"https://host/image:tag"}}},"BaseTgiContainer":{"type":"object","properties":{"maxBatchPrefillTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of prefill tokens used during continuous batching","example":2048,"minimum":0},"maxBatchTotalTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens before moving requests to the next batch","example":2048,"minimum":0},"maxInputLength":{"type":["integer","null"],"format":"int32","description":"Maximum input length by query","example":1024,"minimum":0},"maxTotalTokens":{"type":["integer","null"],"format":"int32","description":"Maximum number of tokens by query","example":1512,"minimum":0}}}}}}