curl --request POST \
--url https://api.benchgen.com/api/competitions/{id}/update_from_repo/ \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"repo": "<string>",
"ref": "<string>",
"path": "<string>",
"task": "<string>",
"bundle_task": 123
}
'import requests
url = "https://api.benchgen.com/api/competitions/{id}/update_from_repo/"
payload = {
"repo": "<string>",
"ref": "<string>",
"path": "<string>",
"task": "<string>",
"bundle_task": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
repo: '<string>',
ref: '<string>',
path: '<string>',
task: '<string>',
bundle_task: 123
})
};
fetch('https://api.benchgen.com/api/competitions/{id}/update_from_repo/', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.benchgen.com/api/competitions/{id}/update_from_repo/",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'repo' => '<string>',
'ref' => '<string>',
'path' => '<string>',
'task' => '<string>',
'bundle_task' => 123
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.benchgen.com/api/competitions/{id}/update_from_repo/"
payload := strings.NewReader("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\",\n \"task\": \"<string>\",\n \"bundle_task\": 123\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.benchgen.com/api/competitions/{id}/update_from_repo/")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\",\n \"task\": \"<string>\",\n \"bundle_task\": 123\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.benchgen.com/api/competitions/{id}/update_from_repo/")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\",\n \"task\": \"<string>\",\n \"bundle_task\": 123\n}"
response = http.request(request)
puts response.read_body{
"valid": true,
"id": 123,
"tasks": [
123
],
"files": [
{}
],
"warnings": [
"<string>"
],
"replaced": [
"<string>"
],
"runs": {
"finished": 123,
"scored_under_previous_files": [
{
"id": 123,
"model_name": "<string>",
"owner": "<string>",
"created_when": "2023-11-07T05:31:56Z"
}
]
}
}{
"valid": true,
"errors": [
"<string>"
]
}Update a benchmark from a repository
Apply the files in a public repository to a benchmark that already exists. This is the other half of keeping a benchmark in git: a change made as a commit, such as widening an answer key, becomes the benchmark’s scoring on the platform. The fetch and the checks are the ones create_from_repo makes, and the install is the one replace_task_files makes, so a repository is not a weaker way in. The bundle is split into its parts by reading competition.yaml, and a part the bundle does not carry bytes for is left alone and reported in skipped, never guessed at. Existing runs are never re-scored: the answer lists the finished runs scored before this change so the caller can offer to run those models again. Applying is always asked for, never a consequence of a push, so an edit made in the web app cannot be overwritten by a repository nobody has looked at. Add ?dry_run=1 to see what would change, reported as would_replace, without changing anything. Requires the benchmark:edit scope; only the creator or a collaborator may do it.
curl --request POST \
--url https://api.benchgen.com/api/competitions/{id}/update_from_repo/ \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"repo": "<string>",
"ref": "<string>",
"path": "<string>",
"task": "<string>",
"bundle_task": 123
}
'import requests
url = "https://api.benchgen.com/api/competitions/{id}/update_from_repo/"
payload = {
"repo": "<string>",
"ref": "<string>",
"path": "<string>",
"task": "<string>",
"bundle_task": 123
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
repo: '<string>',
ref: '<string>',
path: '<string>',
task: '<string>',
bundle_task: 123
})
};
fetch('https://api.benchgen.com/api/competitions/{id}/update_from_repo/', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.benchgen.com/api/competitions/{id}/update_from_repo/",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'repo' => '<string>',
'ref' => '<string>',
'path' => '<string>',
'task' => '<string>',
'bundle_task' => 123
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.benchgen.com/api/competitions/{id}/update_from_repo/"
payload := strings.NewReader("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\",\n \"task\": \"<string>\",\n \"bundle_task\": 123\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.benchgen.com/api/competitions/{id}/update_from_repo/")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\",\n \"task\": \"<string>\",\n \"bundle_task\": 123\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.benchgen.com/api/competitions/{id}/update_from_repo/")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"repo\": \"<string>\",\n \"ref\": \"<string>\",\n \"path\": \"<string>\",\n \"task\": \"<string>\",\n \"bundle_task\": 123\n}"
response = http.request(request)
puts response.read_body{
"valid": true,
"id": 123,
"tasks": [
123
],
"files": [
{}
],
"warnings": [
"<string>"
],
"replaced": [
"<string>"
],
"runs": {
"finished": 123,
"scored_under_previous_files": [
{
"id": 123,
"model_name": "<string>",
"owner": "<string>",
"created_when": "2023-11-07T05:31:56Z"
}
]
}
}{
"valid": true,
"errors": [
"<string>"
]
}ref and a path.
curl -X POST "https://api.benchgen.com/api/competitions/97/update_from_repo/?dry_run=1" \
-H "Authorization: Bearer $BENCHGEN_API_TOKEN" \
-H "Content-Type: application/json" \
-d '{"repo": "https://github.com/acme/benchmarks/tree/main/benchmarks/horses"}'
What it replaces, and what it leaves alone
The bundle is split into the four parts a benchmark is made of: the scoring program, the ingestion program, the questions and the answers. Where each one lives is read fromcompetition.yaml, not assumed, so a bundle that lays its
files out differently still works.
A part the bundle carries no bytes for is left alone and reported, never
guessed at. That happens when competition.yaml does not declare it, or when
it points at a dataset already on the platform rather than a folder. The answer
names those in skipped.
runs.scored_under_previous_files, so
you can offer to run those models again. Nothing is re-run for you, because a
one line change to an answer key must not quietly spend a model run per entry
on the leaderboard.Nothing happens by itself
Applying is always a request you make. There is no webhook, and merging a pull request does not change a benchmark. That is deliberate: a repository nobody has looked at should not be able to overwrite an edit somebody made in the web app.?dry_run=1 first. It answers would_replace with the parts that differ,
and the runs the change would leave scored under previous files, without
changing anything.Private repositories
The platform fetches the archive itself and holds no credentials, so only public repositories work. A private one answers400 with a plain explanation.
If your benchmarks live in a private repository, the BenchGen AI agent can
still apply them: it has its own working copy and sends the files up, rather
than asking the platform to fetch. See versioning a benchmark in
git.
Related
Create from a repository
Replace benchmark files
Authorizations
Platform API token created under Profile Settings > Platform API tokens. Scopes are fixed at creation.
Path Parameters
The benchmark id.
Query Parameters
Validate only; change nothing.
1 Body
The repository URL, for example https://github.com/acme/benchmarks. A browse URL carrying a branch and folder is accepted.
Branch, tag or commit. Defaults to the repository's default branch.
The folder inside the repository holding competition.yaml.
Which of this benchmark's tasks to change, when it has more than one. The id from task_files.
Which task in the bundle to take the files from, when competition.yaml declares more than one. Zero based.