curl --request POST \
--url https://api.benchgen.com/api/competitions/create_from_repo/ \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"repo": "<string>",
"ref": "<string>",
"path": "<string>"
}
'import requests
url = "https://api.benchgen.com/api/competitions/create_from_repo/"
payload = {
"repo": "<string>",
"ref": "<string>",
"path": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({repo: '<string>', ref: '<string>', path: '<string>'})
};
fetch('https://api.benchgen.com/api/competitions/create_from_repo/', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.benchgen.com/api/competitions/create_from_repo/",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'repo' => '<string>',
'ref' => '<string>',
'path' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.benchgen.com/api/competitions/create_from_repo/"
payload := strings.NewReader("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.benchgen.com/api/competitions/create_from_repo/")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.benchgen.com/api/competitions/create_from_repo/")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"valid": true,
"title": "<string>",
"files": 123,
"bytes": 123,
"unpacked_bytes": 123,
"warnings": [
"<string>"
]
}{
"valid": true,
"title": "<string>",
"files": 123,
"bytes": 123,
"unpacked_bytes": 123,
"status_id": 123,
"warnings": [
"<string>"
]
}{
"valid": true,
"errors": [
"<string>"
]
}Create a benchmark from a repository
Build a benchmark from a bundle that lives in a public git repository, so the benchmark has a history: a commit to point at when someone asks which version of it a score was measured against. Send JSON naming the repository, optionally a branch, tag or commit in ref, and optionally the folder inside the repository in path when one repository holds several benchmarks. A browse URL works too, so https://github.com/acme/benchmarks/tree/main/benchmarks/97-horses needs no ref or path. The platform reads the owner, repository and ref out of the URL and builds the archive URL itself against an allowed host (github.com and gitlab.com by default), so the fetch cannot be pointed at an address of the caller’s choosing. A git host wraps its archive in one top-level folder; that wrapper and the path folder are stripped, so competition.yaml ends up in the root exactly as an upload needs it. From there the bundle meets the same checks as create_from_bundle and is reported through creation_status. Private repositories are not supported yet and answer 400. Requires the benchmark:create scope and charges the same creation limits; a dry run does not count.
curl --request POST \
--url https://api.benchgen.com/api/competitions/create_from_repo/ \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"repo": "<string>",
"ref": "<string>",
"path": "<string>"
}
'import requests
url = "https://api.benchgen.com/api/competitions/create_from_repo/"
payload = {
"repo": "<string>",
"ref": "<string>",
"path": "<string>"
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({repo: '<string>', ref: '<string>', path: '<string>'})
};
fetch('https://api.benchgen.com/api/competitions/create_from_repo/', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.benchgen.com/api/competitions/create_from_repo/",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'repo' => '<string>',
'ref' => '<string>',
'path' => '<string>'
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.benchgen.com/api/competitions/create_from_repo/"
payload := strings.NewReader("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\"\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.benchgen.com/api/competitions/create_from_repo/")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\"\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.benchgen.com/api/competitions/create_from_repo/")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\"\n}"
response = http.request(request)
puts response.read_body{
"valid": true,
"title": "<string>",
"files": 123,
"bytes": 123,
"unpacked_bytes": 123,
"warnings": [
"<string>"
]
}{
"valid": true,
"title": "<string>",
"files": 123,
"bytes": 123,
"unpacked_bytes": 123,
"status_id": 123,
"warnings": [
"<string>"
]
}{
"valid": true,
"errors": [
"<string>"
]
}ref and path are only needed for a bare repository address, or
when one repository holds several benchmarks.
There is a working example to try it against, in
benchgen-ai/benchgen-benchmark-examples:
curl -X POST "https://api.benchgen.com/api/competitions/create_from_repo/?dry_run=1" \
-H "Authorization: Bearer $BENCHGEN_API_TOKEN" \
-H "Content-Type: application/json" \
-d '{"repo": "https://github.com/benchgen-ai/benchgen-benchmark-examples/tree/main/benchmarks/cat-care-basics"}'
cat-care-basics is an open answer benchmark scored on the key points each
answer mentions. Drop the ?dry_run=1 to create it.
What the platform does with the address
It never fetches the URL you send. It reads the owner, the repository and the ref out of it and builds the archive address itself, against a host on its allowlist (github.com and gitlab.com by default), so the request cannot be
aimed somewhere else. A git host wraps its archive in one top-level folder,
while competition.yaml has to sit in the zip root, so that wrapper and the
folder you named are stripped before anything else looks at it.
From there the bundle meets exactly the same checks as an upload, and is
reported through creation status.
400 with a plain
explanation rather than a confusing 404.?dry_run=1 to validate the bundle and see the title the platform would
use, without creating anything. It does not count against your creation
limits.A repository of several benchmarks
One repository can hold many, a folder each. That is how the example repository is laid out:benchmarks/
cat-care-basics/
competition.yaml
pages/overview.md
terms.md
logo.png
ingestion_program/
scoring_program/
phase/input_data/
phase/reference_data/
path at the folder that holds competition.yaml, not at the repository
root. A folder that is not there is answered with the list of folders that are,
so a typo tells you what you meant.Authorizations
Platform API token created under Profile Settings > Platform API tokens. Scopes are fixed at creation.
Query Parameters
Validate only: answers 200 with a summary and creates nothing
Body
The repository URL, for example https://github.com/acme/benchmarks. A browse URL carrying a branch and folder is accepted.
Branch, tag or commit. Defaults to the repository's default branch.
The folder inside the repository holding competition.yaml, when it is not at the root.
Response
dry_run: the bundle is usable; nothing was created