From 0e88d8a24d01e9b4c835193fa53cc06e0418451d Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 17 Jun 2025 12:42:10 +0200 Subject: [PATCH 1/4] workflows/labels: log current API rate_limits This will give us a little bit better insight into how close we're running to the limits. --- .github/workflows/labels.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 9f8427124821..06f205f838e3 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -41,6 +41,11 @@ jobs: - name: Install dependencies run: npm install @actions/artifact + - name: Log current API rate limits + env: + GH_TOKEN: ${{ github.token }} + run: gh api /rate_limit | jq + - name: Labels from API data and Eval results uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 env: From 114b4fcf48b1df8acd21082c89ee84cac704bc76 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 17 Jun 2025 13:38:06 +0200 Subject: [PATCH 2/4] workflows/labels: add bottleneck for throttling This should prevent us from hitting API rate limits, even when we run huge manual jobs. It just takes a bit longer. --- .github/workflows/labels.yml | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 06f205f838e3..d1d846aa9fa1 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -39,7 +39,7 @@ jobs: if: github.event_name != 'schedule' || github.repository_owner == 'NixOS' steps: - name: Install dependencies - run: npm install @actions/artifact + run: npm install @actions/artifact bottleneck - name: Log current API rate limits env: @@ -52,12 +52,34 @@ jobs: UPDATED_WITHIN: ${{ inputs.updatedWithin }} with: script: | + const Bottleneck = require('bottleneck') const path = require('node:path') const { DefaultArtifactClient } = require('@actions/artifact') const { readFile } = require('node:fs/promises') const artifactClient = new DefaultArtifactClient() + // Rate-Limiting and Throttling, see for details: + // https://github.com/octokit/octokit.js/issues/1069#throttling + // https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api + const allLimits = new Bottleneck({ + // Avoid concurrent requests + maxConcurrent: 1, + // Hourly limit is at 5000, but other jobs need some, too! + // https://docs.github.com/en/rest/using-the-rest-api/rate-limits-for-the-rest-api + reservoir: 1000, + reservoirRefreshAmount: 1000, + reservoirRefreshInterval: 60 * 60 * 1000 + }) + // Pause between mutative requests + const writeLimits = new Bottleneck({ minTime: 1000 }).chain(allLimits) + github.hook.wrap('request', async (request, options) => { + if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(options.method)) + return writeLimits.schedule(request.bind(null, options)) + else + return allLimits.schedule(request.bind(null, options)) + }) + if (process.env.UPDATED_WITHIN && !/^\d+$/.test(process.env.UPDATED_WITHIN)) throw new Error('Please enter "updated within" as integer in hours.') From f3b67f4eb5a1f757a0e784f6197e14a9d9cbcd80 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 17 Jun 2025 14:01:45 +0200 Subject: [PATCH 3/4] workflows/labels: improve logging for skipped PRs Conditions that cause a PR to be skipped are now marked clearly in the log output. --- .github/workflows/labels.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index d1d846aa9fa1..ce126788ea69 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -128,10 +128,13 @@ jobs: }, async (response, done) => (await Promise.allSettled(response.data.map(async (pull_request) => { try { - const log = (k,v) => core.info(`PR #${pull_request.number} - ${k}: ${v}`) + const log = (k,v,skip) => { + core.info(`PR #${pull_request.number} - ${k}: ${v}` + (skip ? ' (skipped)' : '')) + return skip + } - log('Last updated at', pull_request.updated_at) - if (new Date(pull_request.updated_at) < cutoff) return done() + if (log('Last updated at', pull_request.updated_at, new Date(pull_request.updated_at) < cutoff)) + return done() log('URL', pull_request.html_url) const run_id = (await github.rest.actions.listWorkflowRuns({ @@ -146,8 +149,8 @@ jobs: // Newer PRs might not have run Eval to completion, yet. We can skip them, because this // job will be run as part of that Eval run anyway. - log('Last eval run', run_id ?? '') - if (!run_id) return; + if (log('Last eval run', run_id ?? '', !run_id)) + return; const artifact = (await github.rest.actions.listWorkflowRunArtifacts({ ...context.repo, @@ -159,8 +162,9 @@ jobs: // actually download the artifact in the next step and avoid that race condition. // Older PRs, where the workflow run was already eval.yml, but the artifact was not // called "comparison", yet, will be skipped as well. - log('Artifact expires at', artifact?.expires_at ?? '') - if (new Date(artifact?.expires_at ?? 0) < new Date(new Date().getTime() + 60 * 1000)) return; + const expired = new Date(artifact?.expires_at ?? 0) < new Date(new Date().getTime() + 60 * 1000) + if (log('Artifact expires at', artifact?.expires_at ?? '', expired)) + return; await artifactClient.downloadArtifact(artifact.id, { findBy: { From 4fe9129b789845c25246b77bc76e9a217919a488 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 17 Jun 2025 14:44:07 +0200 Subject: [PATCH 4/4] workflows/labels: add simple stats To ease debugging rate-limiting issues. --- .github/workflows/labels.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index ce126788ea69..2ae77cf01421 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -59,6 +59,12 @@ jobs: const artifactClient = new DefaultArtifactClient() + const stats = { + prs: 0, + requests: 0, + artifacts: 0 + } + // Rate-Limiting and Throttling, see for details: // https://github.com/octokit/octokit.js/issues/1069#throttling // https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api @@ -74,6 +80,7 @@ jobs: // Pause between mutative requests const writeLimits = new Bottleneck({ minTime: 1000 }).chain(allLimits) github.hook.wrap('request', async (request, options) => { + stats.requests++ if (['POST', 'PUT', 'PATCH', 'DELETE'].includes(options.method)) return writeLimits.schedule(request.bind(null, options)) else @@ -117,7 +124,7 @@ jobs: base: context.payload.pull_request.base.ref } - await github.paginate( + const prs = await github.paginate( github.rest.pulls.list, { ...context.repo, @@ -126,7 +133,7 @@ jobs: direction: 'desc', ...prEventCondition }, - async (response, done) => (await Promise.allSettled(response.data.map(async (pull_request) => { + (response, done) => response.data.map(async (pull_request) => { try { const log = (k,v,skip) => { core.info(`PR #${pull_request.number} - ${k}: ${v}` + (skip ? ' (skipped)' : '')) @@ -135,6 +142,7 @@ jobs: if (log('Last updated at', pull_request.updated_at, new Date(pull_request.updated_at) < cutoff)) return done() + stats.prs++ log('URL', pull_request.html_url) const run_id = (await github.rest.actions.listWorkflowRuns({ @@ -165,6 +173,7 @@ jobs: const expired = new Date(artifact?.expires_at ?? 0) < new Date(new Date().getTime() + 60 * 1000) if (log('Artifact expires at', artifact?.expires_at ?? '', expired)) return; + stats.artifacts++ await artifactClient.downloadArtifact(artifact.id, { findBy: { @@ -226,10 +235,14 @@ jobs: } catch (cause) { throw new Error(`Labeling PR #${pull_request.number} failed.`, { cause }) } - }))) + }) + ); + + (await Promise.allSettled(prs.flat())) .filter(({ status }) => status == 'rejected') .map(({ reason }) => core.setFailed(`${reason.message}\n${reason.cause.stack}`)) - ) + + core.notice(`Processed ${stats.prs} PRs, made ${stats.requests + stats.artifacts} API requests and downloaded ${stats.artifacts} artifacts.`) - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0 name: Labels from touched files