improve queue

gulpfile.js

@@ -1,62 +1,126 @@
-const { src, dest } = require("gulp");
+const { src, dest, parallel } = require("gulp");
 const uglify = require("gulp-uglify");
 const concat = require("gulp-concat");
 var order = require("gulp-order");
 const cleanCss = require("gulp-clean-css");
+const crypto = require("crypto");
+const fs = require("fs");
+const path = require("path");
 
-function defaultTask(cb) {
+const coreJsFiles = [
-  const jsFiles = [
+  "public/script/external/angular.min.js",
-    "public/script/external/angular.min.js",
+  "public/script/external/angular-translate.min.js",
-    "public/script/external/angular-translate.min.js",
+  "public/script/external/angular-translate-loader-static-files.min.js",
-    "public/script/external/angular-translate-loader-static-files.min.js",
+  "public/script/external/angular-sanitize.min.js",
-    "public/script/external/angular-sanitize.min.js",
+  "public/script/external/angular-route.min.js",
-    "public/script/external/angular-route.min.js",
+  "public/script/external/github-emojis.js",
-    "public/script/external/pdf.compat.js",
+  "public/script/external/marked-emoji.js",
-    "public/script/external/pdf.js",
+  "public/script/external/marked.min.js",
-    "public/script/external/github-emojis.js",
+  "public/script/external/purify.min.js",
-    "public/script/external/marked-emoji.js",
+  "public/script/external/ansi_up.min.js",
-    "public/script/external/marked.min.js",
+  "public/script/external/prism.min.js",
-    "public/script/external/purify.min.js",
+  "public/script/external/jquery-3.4.1.min.js",
-    "public/script/external/ansi_up.min.js",
+  "public/script/external/popper.min.js",
-    "public/script/external/prism.min.js",
+  "public/script/external/bootstrap.min.js",
-    "public/script/external/katex.min.js",
+  "public/script/utils.js",
-    "public/script/external/katex-auto-render.min.js",
+];
-    "public/script/external/marked-katex-extension.umd.min.js",
+
-    "public/script/external/mermaid.min.js",
+const vendorJsFiles = [
-    "public/script/external/marked-mermaid.js",
+  "public/script/external/pdf.compat.js",
-    "public/script/external/notebook.min.js",
+  "public/script/external/pdf.js",
-    "public/script/external/org.js",
+  "public/script/ng-pdfviewer.min.js",
-    "public/script/external/jquery-3.4.1.min.js",
+  "public/script/external/katex.min.js",
-    "public/script/external/popper.min.js",
+  "public/script/external/katex-auto-render.min.js",
-    "public/script/external/bootstrap.min.js",
+  "public/script/external/marked-katex-extension.umd.min.js",
-    "public/script/external/ace/ace.js",
+  "public/script/external/marked-mermaid.js",
-    "public/script/external/ui-ace.min.js",
+  "public/script/external/notebook.min.js",
-    "public/script/utils.js",
+  "public/script/external/org.js",
-    "public/script/ng-pdfviewer.min.js",
+  "public/script/external/ace/ace.js",
-    "public/script/app.js",
+  "public/script/external/ui-ace.min.js",
-    "public/script/admin.js",
+  "public/script/app.js",
-  ];
+  "public/script/admin.js",
-  const cssFiles = [
+];
-    "public/css/bootstrap.min.css",
+
-    "public/css/font-awesome.min.css",
+const mermaidFiles = [
-    "public/css/notebook.css",
+  "public/script/external/mermaid.min.js",
-    "public/css/katex.min.css",
+];
-    "public/css/mermaid.css",
+
-    "public/css/github-markdown.min.css",
+const cssFiles = [
-    "public/css/style.css",
+  "public/css/bootstrap.min.css",
-  ];
+  "public/css/font-awesome.min.css",
-  src(jsFiles)
+  "public/css/notebook.css",
-    .pipe(order(jsFiles, { base: "./" }))
+  "public/css/katex.min.css",
-    .pipe(concat("bundle.min.js"))
+  "public/css/mermaid.css",
+  "public/css/github-markdown.min.css",
+  "public/css/style.css",
+];
+
+function hashFile(filePath) {
+  const content = fs.readFileSync(filePath);
+  return crypto.createHash("md5").update(content).digest("hex").slice(0, 10);
+}
+
+function buildCoreJs(cb) {
+  src(coreJsFiles)
+    .pipe(order(coreJsFiles, { base: "./" }))
+    .pipe(concat("core.min.js"))
     .pipe(uglify())
     .pipe(dest("public/script"))
     .on("end", cb);
+}
 
+function buildVendorJs(cb) {
+  src(vendorJsFiles)
+    .pipe(order(vendorJsFiles, { base: "./" }))
+    .pipe(concat("vendor.min.js"))
+    .pipe(uglify())
+    .pipe(dest("public/script"))
+    .on("end", cb);
+}
+
+function buildMermaidJs(cb) {
+  src(mermaidFiles)
+    .pipe(concat("mermaid.min.js"))
+    .pipe(dest("public/script"))
+    .on("end", cb);
+}
+
+function buildCss(cb) {
   src(cssFiles)
     .pipe(order(cssFiles, { base: "./" }))
     .pipe(concat("all.min.css"))
     .pipe(cleanCss())
-    .pipe(dest("public/css"));
+    .pipe(dest("public/css"))
+    .on("end", cb);
 }
 
-exports.default = defaultTask;
+function writeManifest(cb) {
+  const files = {
+    "core.min.js": "public/script/core.min.js",
+    "vendor.min.js": "public/script/vendor.min.js",
+    "mermaid.min.js": "public/script/mermaid.min.js",
+    "all.min.css": "public/css/all.min.css",
+  };
+  const manifest = {};
+  for (const [key, filePath] of Object.entries(files)) {
+    const hash = hashFile(filePath);
+    // Insert hash before the compound extension: core.min.js → core.HASH.min.js
+    const firstDot = key.indexOf(".");
+    const base = key.slice(0, firstDot);
+    const ext = key.slice(firstDot);
+    manifest[key] = `${base}.${hash}${ext}`;
+  }
+  fs.writeFileSync(
+    "public/asset-manifest.json",
+    JSON.stringify(manifest, null, 2)
+  );
+  cb();
+}
+
+const buildAssets = parallel(buildCoreJs, buildVendorJs, buildMermaidJs, buildCss);
+
+exports.default = function (cb) {
+  buildAssets(function () {
+    writeManifest(cb);
+  });
+};

public/asset-manifest.json

@@ -1,6 +1,6 @@
 {
   "core.min.js": "core.3db744fc07.min.js",
-  "vendor.min.js": "vendor.9df222182a.min.js",
+  "vendor.min.js": "vendor.9aa24967d7.min.js",
   "mermaid.min.js": "mermaid.f848a72d16.min.js",
-  "all.min.css": "all.8d9fbb7ca6.min.css"
+  "all.min.css": "all.f79970ad3b.min.css"
 }

public/css/style.css

@@ -3289,6 +3289,35 @@ code {
   color: var(--color);
 }
 .dark-mode .paper-error-card { border-left-color: #FF8B7B; }
+
+.paper-ratelimit-card {
+  margin-top: 18px;
+  padding: 20px 22px;
+  background: var(--paper-bg-alt);
+  border: 1px solid var(--border-color);
+  border-left: 3px solid #D69E2E;
+  border-radius: 10px;
+  color: var(--color);
+}
+.dark-mode .paper-ratelimit-card { border-left-color: #F6E05E; }
+.paper-ratelimit-head {
+  display: flex;
+  align-items: flex-start;
+  gap: 14px;
+}
+.paper-ratelimit-head > i {
+  font-size: 18px;
+  color: #D69E2E;
+  margin-top: 4px;
+  flex-shrink: 0;
+}
+.dark-mode .paper-ratelimit-head > i { color: #F6E05E; }
+.paper-ratelimit-card strong { color: var(--color); }
+
+.status-pill-ratelimit { border-color: #D69E2E; color: #D69E2E; }
+.dark-mode .status-pill-ratelimit { border-color: #F6E05E; color: #F6E05E; }
+.status-ratelimit { background: #D69E2E; }
+.dark-mode .status-ratelimit { background: #F6E05E; }
 .paper-error-head {
   display: flex;
   align-items: flex-start;
@@ -4681,10 +4710,45 @@ textarea::selection {
   border-radius: 10px;
   padding: 16px 20px;
 }
+.q-chart-wrap {
+  position: relative;
+}
 .q-throughput canvas {
   width: 100%;
   display: block;
 }
+.q-chart-tooltip {
+  position: absolute;
+  pointer-events: none;
+  background: var(--paper-card);
+  border: 1px solid var(--border-color);
+  border-radius: 6px;
+  padding: 6px 10px;
+  font-family: var(--font-mono);
+  font-size: 11px;
+  line-height: 1.6;
+  white-space: nowrap;
+  box-shadow: 0 2px 8px rgba(0,0,0,0.15);
+  z-index: 10;
+}
+.q-chart-tooltip .q-tip-time {
+  color: var(--ink-muted);
+  margin-bottom: 2px;
+}
+.q-chart-tooltip .q-tip-completed { color: #3B4AD6; }
+.q-chart-tooltip .q-tip-failed { color: #B42318; }
+.dark-mode .q-chart-tooltip .q-tip-completed { color: #A7B2FF; }
+.dark-mode .q-chart-tooltip .q-tip-failed { color: #F08A82; }
+.q-chart-crosshair {
+  position: absolute;
+  top: 0;
+  width: 1px;
+  height: 100%;
+  background: var(--ink-muted);
+  opacity: 0.3;
+  pointer-events: none;
+  z-index: 9;
+}
 .q-stats-panel {
   background: var(--paper-card);
   border: 1px solid var(--border-color);
@@ -4708,6 +4772,14 @@ textarea::selection {
   font-size: 10px;
   letter-spacing: 0.08em;
 }
+.q-legend-completed { color: #3B4AD6; }
+.q-legend-failed { color: #B42318; margin-left: 8px; }
+.q-legend-exec { color: #B8860B; margin-left: 8px; letter-spacing: 2px; }
+.dark-mode .q-legend-completed { color: #A7B2FF; }
+.dark-mode .q-legend-failed { color: #F08A82; }
+.dark-mode .q-legend-exec { color: #F5C842; }
+.q-chart-tooltip .q-tip-exec { color: #B8860B; }
+.dark-mode .q-chart-tooltip .q-tip-exec { color: #F5C842; }
 .q-stats-grid {
   display: grid;
   grid-template-columns: 1fr 1fr;
@@ -4761,25 +4833,67 @@ textarea::selection {
   gap: 10px;
   margin-bottom: 12px;
 }
-.q-tabs {
+/* State filter toggles */
+.q-state-filters {
   display: inline-flex;
-  gap: 4px;
+  gap: 6px;
+  flex-wrap: wrap;
 }
-.q-tab {
+.q-state-toggle {
+  cursor: pointer;
+  display: inline-flex;
+  align-items: center;
+}
+.q-state-toggle input { display: none; }
+.q-state-chip {
   font-family: var(--font-mono);
   font-size: 11px;
-  padding: 5px 14px;
+  padding: 3px 10px;
-  border-radius: 8px;
+  border-radius: 10px;
   border: 1px solid var(--border-color);
   background: var(--paper-card);
   color: var(--ink-muted);
-  cursor: pointer;
+  transition: opacity 0.15s, background 0.15s;
-  transition: background 0.12s, color 0.12s;
+  text-transform: capitalize;
 }
-.q-tab:hover { background: var(--paper-bg-alt); }
+.q-state-toggle input:checked + .q-state-chip {
-.q-tab.active {
+  opacity: 1;
-  background: var(--color);
+}
-  color: var(--paper-card);
+.q-state-toggle input:not(:checked) + .q-state-chip {
+  opacity: 0.4;
+  text-decoration: line-through;
+}
+
+/* State badge colors */
+.q-state-badge, .q-state-chip {
+  font-family: var(--font-mono);
+  font-size: 10px;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: 0.3px;
+}
+.q-state-badge {
+  display: inline-block;
+  padding: 2px 8px;
+  border-radius: 8px;
+}
+.q-state-active  { background: #dbeafe; color: #1d4ed8; }
+.q-state-waiting { background: #fef3c7; color: #92400e; }
+.q-state-delayed { background: #e0e7ff; color: #4338ca; }
+.q-state-failed  { background: #fee2e2; color: #b91c1c; }
+.q-state-completed { background: #dcfce7; color: #166534; }
+.dark-mode .q-state-active  { background: rgba(59,130,246,0.2); color: #93bbfd; }
+.dark-mode .q-state-waiting { background: rgba(251,191,36,0.15); color: #fbbf24; }
+.dark-mode .q-state-delayed { background: rgba(99,102,241,0.2); color: #a5b4fc; }
+.dark-mode .q-state-failed  { background: rgba(239,68,68,0.2); color: #fca5a5; }
+.dark-mode .q-state-completed { background: rgba(34,197,94,0.15); color: #86efac; }
+.q-cell-state { width: 120px; }
+.q-delay-hint {
+  display: block;
+  font-size: 10px;
+  color: var(--ink-muted);
+  margin-top: 2px;
+  font-family: var(--font-mono);
 }
 .q-search-row {
   display: flex;
@@ -4835,6 +4949,20 @@ textarea::selection {
   font-size: 12px;
   white-space: nowrap;
 }
+.q-cell-id {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+.q-chevron {
+  font-size: 9px;
+  color: var(--ink-muted);
+  transition: transform 0.15s;
+  flex-shrink: 0;
+}
+.q-chevron-open {
+  transform: rotate(90deg);
+}
 .q-cell-id a { color: var(--color); }
 .q-cell-payload {
   font-family: var(--font-mono);
@@ -4901,11 +5029,94 @@ textarea::selection {
   margin-bottom: 4px;
 }
 .dark-mode .q-error-reason { color: #F08A82; }
+/* Error toast */
+.q-toast-error {
+  background: #FBE7E7;
+  color: #B42318;
+  border: 1px solid #F3C7C7;
+  border-radius: 8px;
+  padding: 10px 16px;
+  margin-bottom: 12px;
+  font-family: var(--font-mono);
+  font-size: 12px;
+  cursor: pointer;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+.dark-mode .q-toast-error {
+  background: rgba(240,138,130,0.1);
+  color: #F08A82;
+  border-color: rgba(240,138,130,0.25);
+}
+
 .q-error-stack {
   font-size: 0.78rem;
-  max-height: 80px;
+  max-height: 120px;
   overflow: auto;
   margin: 4px 0 0;
+  background: var(--paper-bg-alt);
+  border: 1px solid var(--border-color);
+  border-radius: 6px;
+  padding: 8px 10px;
+}
+
+/* Expanded job detail row */
+.q-row-expanded > td { border-bottom-color: transparent; }
+.q-detail-row > td {
+  padding: 0 10px 14px !important;
+  border-bottom: 1px solid var(--border-color);
+}
+.q-job-detail {
+  background: var(--paper-bg-alt);
+  border: 1px solid var(--border-color);
+  border-radius: 8px;
+  padding: 14px 16px;
+}
+.q-job-detail-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fill, minmax(160px, 1fr));
+  gap: 10px 20px;
+  margin-bottom: 10px;
+}
+.q-job-detail-item {
+  display: flex;
+  flex-direction: column;
+  gap: 2px;
+}
+.q-job-detail-label {
+  font-family: var(--font-mono);
+  font-size: 9px;
+  font-weight: 500;
+  color: var(--ink-muted);
+  text-transform: uppercase;
+  letter-spacing: 0.08em;
+}
+.q-job-detail-value {
+  font-family: var(--font-mono);
+  font-size: 12px;
+  color: var(--color);
+  word-break: break-all;
+}
+.q-job-detail-value a { color: var(--color); }
+.q-job-detail-error {
+  margin: 8px 0;
+}
+.q-job-detail-actions {
+  display: flex;
+  gap: 8px;
+  margin-top: 12px;
+  padding-top: 10px;
+  border-top: 1px solid var(--border-color);
+}
+.q-job-detail-actions .btn {
+  font-family: var(--font-mono);
+  font-size: 11px;
+  padding: 5px 12px;
+  border-radius: 6px;
+  background: var(--paper-card);
+  border: 1px solid var(--border-color);
+  color: var(--color);
 }
 
 @media (max-width: 900px) {
@@ -4918,6 +5129,7 @@ textarea::selection {
   .q-search-row .form-control { max-width: 100%; flex: 1; }
   .q-table { font-size: 12px; }
   .q-header { flex-direction: column; }
+  .q-job-detail-grid { grid-template-columns: 1fr 1fr; }
 }
 
 @media (max-width: 900px) {

public/i18n/locale-en.json

@@ -23,6 +23,7 @@
     "repoUrl_not_defined": "The repository URL needs to be defined.",
     "source_not_provided": "A repository source must be provided.",
     "github_rate_limit_exceeded": "GitHub temporarily blocked the request because we hit its API rate limit. Wait a few minutes and try again. If the problem persists and you contact GitHub Support, include the request ID and timestamp shown in GitHub's response (the 'X-GitHub-Request-Id' header).",
+    "rate_limited": "GitHub API rate limit reached. The repository will be available shortly — please wait a moment and refresh.",
     "repoId_already_used": "The repository ID is already used.",
     "invalid_repoId": "The format of the repository ID is invalid.",
     "unsupported_source": "The repository source type is not supported.",

public/partials/admin/queues.htm

@@ -44,8 +44,12 @@
   <!-- Detail: throughput chart + stats panel -->
   <div class="q-detail" ng-if="selectedStats">
     <div class="q-throughput">
-      <div class="q-section-label">{{selectedQueue}}&middot;throughput <span class="q-section-right">COMPLETED / MIN &middot; {{range | uppercase}}</span></div>
+      <div class="q-section-label">{{selectedQueue}}&middot;throughput <span class="q-section-right"><span class="q-legend-completed">&#9679;</span> completed <span class="q-legend-failed">&#9679;</span> failed <span class="q-legend-exec">- -</span> avg time &middot; {{range | uppercase}}</span></div>
-      <canvas id="q-throughput-chart" height="180"></canvas>
+      <div class="q-chart-wrap">
+        <canvas id="q-throughput-chart" height="180"></canvas>
+        <div id="q-chart-tooltip" class="q-chart-tooltip" style="display:none;"></div>
+        <div id="q-chart-crosshair" class="q-chart-crosshair" style="display:none;"></div>
+      </div>
     </div>
     <div class="q-stats-panel">
       <div class="q-section-label">{{selectedQueue}}&middot;stats</div>
@@ -83,16 +87,19 @@
     </div>
   </div>
 
+  <div class="q-toast-error" ng-if="actionError" ng-click="actionError = null">
+    <i class="fas fa-exclamation-circle"></i> {{actionError}}
+  </div>
+
   <!-- Jobs table -->
   <div class="q-jobs">
     <div class="q-jobs-header">
-      <div class="q-section-label">{{query.state | uppercase}} JOBS &middot; {{selectedQueue | uppercase}}</div>
+      <div class="q-section-label">ALL JOBS &middot; {{selectedQueue | uppercase}}</div>
-      <div class="q-tabs">
+      <div class="q-state-filters">
-        <button class="q-tab" ng-class="{active: query.state == 'active'}" ng-click="query.state = 'active'">Active</button>
+        <label class="q-state-toggle" ng-repeat="s in allStates">
-        <button class="q-tab" ng-class="{active: query.state == 'waiting'}" ng-click="query.state = 'waiting'">Waiting</button>
+          <input type="checkbox" ng-model="stateFilter[s]" />
-        <button class="q-tab" ng-class="{active: query.state == 'completed'}" ng-click="query.state = 'completed'">Completed</button>
+          <span class="q-state-chip q-state-{{s}}">{{s}}</span>
-        <button class="q-tab" ng-class="{active: query.state == 'failed'}" ng-click="query.state = 'failed'">Failed</button>
+        </label>
-        <button class="q-tab" ng-class="{active: query.state == 'delayed'}" ng-click="query.state = 'delayed'">Delayed</button>
       </div>
     </div>
 
@@ -105,9 +112,10 @@
       <button class="btn btn-sm" type="button" ng-click="refreshNow()" title="Refresh now"><i class="fas fa-sync"></i></button>
     </div>
 
-    <table class="q-table" ng-if="jobs.length > 0">
+    <table class="q-table" ng-if="filteredJobs().length > 0">
       <thead>
         <tr>
+          <th>STATE</th>
           <th>JOB ID</th>
           <th>PAYLOAD</th>
           <th>ATTEMPTS</th>
@@ -116,14 +124,19 @@
           <th></th>
         </tr>
       </thead>
-      <tbody>
+      <tbody ng-repeat="job in filteredJobs()">
-        <tr ng-repeat="job in jobs" ng-class="{'q-row-failed': job.failedReason || job.stacktrace.length}">
+        <tr ng-class="{'q-row-failed': job._state == 'failed', 'q-row-expanded': expanded[job.id]}" ng-click="toggleJob(job)" style="cursor:pointer;">
+          <td class="q-cell-state">
+            <span class="q-state-badge q-state-{{job._state}}" ng-bind="job._state"></span>
+            <span class="q-delay-hint" ng-if="job._state == 'delayed' && job.delayUntil" ng-bind="delayCountdown(job.delayUntil)"></span>
+          </td>
           <td class="q-cell-id">
-            <a target="_blank" ng-href="/r/{{job.id}}" ng-bind="'job:' + (job.id | limitTo:6)"></a>
+            <i class="fas fa-chevron-right q-chevron" ng-class="{'q-chevron-open': expanded[job.id]}"></i>
+            <a target="_blank" ng-href="/r/{{job.id}}" ng-click="$event.stopPropagation()" ng-bind="'job:' + (job.id | limitTo:6)"></a>
           </td>
           <td class="q-cell-payload">
             <span ng-bind="job.name || 'anonymize'"></span>
-            <span class="q-payload-detail" ng-if="job.data.repoId"> &middot; {{job.data.repoId}}</span>
+            <span class="q-payload-detail" ng-if="job.data.repoId && job.data.repoId !== job.name"> &middot; {{job.data.repoId}}</span>
           </td>
           <td class="q-cell-num" ng-bind="job.attemptsMade || 1"></td>
           <td class="q-cell-num" ng-bind="jobDuration(job)"></td>
@@ -134,22 +147,77 @@
             </div>
           </td>
           <td class="q-cell-actions">
-            <button class="btn btn-sm" ng-click="retryJob(job)" title="Retry"><i class="fas fa-sync"></i></button>
+            <button class="btn btn-sm" ng-click="retryJob(job); $event.stopPropagation()" title="Retry" ng-if="job._state == 'failed'"><i class="fas fa-sync"></i></button>
-            <button class="btn btn-sm" ng-click="removeJob(job)" title="Remove"><i class="fas fa-trash-alt"></i></button>
+            <button class="btn btn-sm" ng-click="removeJob(job); $event.stopPropagation()" title="Remove"><i class="fas fa-trash-alt"></i></button>
+          </td>
+        </tr>
+        <tr class="q-detail-row" ng-if="expanded[job.id]">
+          <td colspan="7">
+            <div class="q-job-detail">
+              <div class="q-job-detail-grid">
+                <div class="q-job-detail-item">
+                  <span class="q-job-detail-label">JOB ID</span>
+                  <span class="q-job-detail-value"><a target="_blank" ng-href="/r/{{job.id}}" ng-bind="job.id"></a></span>
+                </div>
+                <div class="q-job-detail-item">
+                  <span class="q-job-detail-label">STATE</span>
+                  <span class="q-job-detail-value"><span class="q-state-badge q-state-{{job._state}}" ng-bind="job._state"></span></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job.data.repoId">
+                  <span class="q-job-detail-label">REPO ID</span>
+                  <span class="q-job-detail-value" ng-bind="job.data.repoId"></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job.timestamp">
+                  <span class="q-job-detail-label">CREATED</span>
+                  <span class="q-job-detail-value" ng-bind="humanTime(job.timestamp)"></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job._state == 'delayed' && job.delayUntil">
+                  <span class="q-job-detail-label">RETRY AT</span>
+                  <span class="q-job-detail-value">{{humanTime(job.delayUntil)}} ({{delayCountdown(job.delayUntil)}})</span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job.processedOn">
+                  <span class="q-job-detail-label">PROCESSED</span>
+                  <span class="q-job-detail-value" ng-bind="humanTime(job.processedOn)"></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job.finishedOn">
+                  <span class="q-job-detail-label">FINISHED</span>
+                  <span class="q-job-detail-value" ng-bind="humanTime(job.finishedOn)"></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job.attemptsMade">
+                  <span class="q-job-detail-label">ATTEMPTS</span>
+                  <span class="q-job-detail-value" ng-bind="job.attemptsMade"></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="job.progress && job.progress.status">
+                  <span class="q-job-detail-label">STATUS</span>
+                  <span class="q-job-detail-value" ng-bind="job.progress.status"></span>
+                </div>
+                <div class="q-job-detail-item" ng-if="jobProgressPct(job) !== null">
+                  <span class="q-job-detail-label">PROGRESS</span>
+                  <span class="q-job-detail-value" ng-bind="jobProgressPct(job) + '%'"></span>
+                </div>
+              </div>
+              <div ng-if="job.failedReason" class="q-job-detail-error">
+                <span class="q-job-detail-label">ERROR</span>
+                <div class="q-error-reason" ng-bind="job.failedReason"></div>
+              </div>
+              <div ng-if="job.stacktrace.length">
+                <span class="q-job-detail-label">STACKTRACE</span>
+                <pre ng-repeat="stack in job.stacktrace track by $index" class="q-error-stack"><code ng-bind="stack"></code></pre>
+              </div>
+              <div class="q-job-detail-actions">
+                <button class="btn btn-sm" ng-click="retryJob(job)" ng-if="job._state == 'failed'"><i class="fas fa-sync"></i> Retry</button>
+                <button class="btn btn-sm" ng-click="removeJob(job)"><i class="fas fa-trash-alt"></i> Remove</button>
+                <a class="btn btn-sm" target="_blank" ng-href="/r/{{job.id}}"><i class="fas fa-external-link-alt"></i> View repo</a>
+              </div>
+            </div>
           </td>
         </tr>
       </tbody>
     </table>
 
-    <!-- Expanded error detail -->
+    <div class="paper-table-empty" ng-if="filteredJobs().length == 0" style="border:1px solid var(--border-color);border-radius:10px;background:var(--paper-card);">
-    <div ng-repeat="job in jobs" ng-if="job.failedReason || job.stacktrace.length" class="q-error-detail" style="display:none;">
-      <div ng-if="job.failedReason" class="q-error-reason" ng-bind="job.failedReason"></div>
-      <pre ng-repeat="stack in job.stacktrace track by $index" class="q-error-stack"><code ng-bind="stack"></code></pre>
-    </div>
-
-    <div class="paper-table-empty" ng-if="jobs.length == 0" style="border:1px solid var(--border-color);border-radius:10px;background:var(--paper-card);">
       <i class="fas fa-check-circle"></i>
-      <span ng-if="!query.search">No {{query.state}} jobs in the {{selectedQueue}} queue.</span>
+      <span ng-if="!query.search">No jobs in the {{selectedQueue}} queue.</span>
       <span ng-if="query.search">No jobs match the current filters.</span>
     </div>
   </div>

public/partials/admin/repositories.htm

@@ -103,7 +103,7 @@
           <span class="status-dot" ng-class="{'status-removed': repo.status == 'removed' || repo.status == 'expired', 'status-ready': repo.status == 'ready', 'status-error': repo.status == 'error', 'status-preparing': repo.status == 'preparing'}"></span>
           <span ng-bind="repo.status | title"></span>
         </span>
-        <span class="status-sub" ng-if="repo.statusMessage" title="{{repo.statusMessage}}" ng-bind="repo.statusMessage"></span>
+        <span class="status-sub" ng-if="repo.statusMessage" title="{{repo.statusMessage}}" ng-bind="repo.statusMessage | statusMsg"></span>
       </div>
       <div class="cell-views num" role="cell" ng-bind="::repo.pageView || 0 | number"></div>
       <div class="cell-expires" role="cell" ng-bind="repo.anonymizeDate | humanTime"></div>

public/partials/admin/user.htm

@@ -201,7 +201,7 @@
           <span class="status-dot" ng-class="{'status-removed': repo.status == 'removed' || repo.status == 'expired', 'status-ready': repo.status == 'ready', 'status-error': repo.status == 'error', 'status-preparing': repo.status == 'preparing'}"></span>
           <span ng-bind="repo.status | title"></span>
         </span>
-        <span class="status-sub" ng-if="repo.statusMessage" title="{{repo.statusMessage}}" ng-bind="repo.statusMessage"></span>
+        <span class="status-sub" ng-if="repo.statusMessage" title="{{repo.statusMessage}}" ng-bind="repo.statusMessage | statusMsg"></span>
       </div>
       <div class="cell-views num" role="cell" ng-bind="::repo.pageView || 0 | number"></div>
       <div class="cell-expires" role="cell" ng-bind="repo.anonymizeDate | humanTime"></div>

public/partials/dashboard.htm

@@ -206,7 +206,7 @@
             ></span>
             <span ng-bind="item.status | title"></span>
           </div>
-          <div class="status-sub status-sub-error" ng-if="item.status == 'error' && item.statusMessage" title="{{item.statusMessage}}" ng-bind="item.statusMessage"></div>
+          <div class="status-sub status-sub-error" ng-if="item.status == 'error' && item.statusMessage" title="{{item.statusMessage}}" ng-bind="item.statusMessage | statusMsg"></div>
           <div class="status-sub" ng-if="item.status != 'error' && item.anonymizeDate" title="Last anonymized {{item.anonymizeDate | humanTime}}" ng-bind="item.anonymizeDate | humanTime"></div>
         </div>
         <div class="cell-views num" role="cell" ng-bind="item.pageView | number"></div>

public/partials/pageView.htm

@@ -8,6 +8,18 @@
 </div>
 <div ng-if="type == 'audio'"><audio controls="controls"><source ng-src="{{url}}" /></audio></div>
 <div ng-if="type == 'IPython'"><notebook file="url"></notebook></div>
+<div ng-if="type == 'rate_limited'" class="file-error container d-flex h-100">
+  <div class="paper-ratelimit-card m-auto" style="max-width:520px;">
+    <div class="paper-ratelimit-head">
+      <i class="fas fa-hourglass-half"></i>
+      <div>
+        <div class="paper-error-eyebrow">Temporarily paused</div>
+        <div class="paper-error-title">GitHub API rate limit reached</div>
+      </div>
+    </div>
+    <p class="paper-error-msg">This repository will be available in <strong>{{rateLimitCountdown}}</strong>. The page will reload automatically.</p>
+  </div>
+</div>
 <div ng-if="type == 'error'" class="file-error container d-flex h-100"><h1 class="paper-empty-title m-auto" translate="ERRORS.{{content}}">Error</h1></div>
 <div ng-if="type == 'loading' && !error" class="file-error container d-flex h-100"><h1 class="paper-empty-title m-auto">Loading&hellip;</h1></div>
 <div ng-if="type == 'empty'" class="file-error container d-flex h-100"><h1 class="paper-empty-title m-auto">Empty <em>repository</em>.</h1></div>

public/partials/status.htm

@@ -5,9 +5,10 @@
       <h1 class="paper-page-title">Status of <em>{{repoId}}</em></h1>
       <p class="paper-page-lede">Track progress as your anonymization is prepared.</p>
     </div>
-    <span class="status-pill" ng-class="{'status-pill-ready': repo.status == 'ready', 'status-pill-error': repo.status == 'error', 'status-pill-removed': repo.status == 'removed' || repo.status == 'expired'}">
+    <span class="status-pill" ng-class="{'status-pill-ready': repo.status == 'ready', 'status-pill-error': repo.status == 'error', 'status-pill-removed': repo.status == 'removed' || repo.status == 'expired', 'status-pill-ratelimit': rateLimitResetAt}">
-      <span class="status-dot" ng-class="{'status-ready': repo.status == 'ready', 'status-error': repo.status == 'error', 'status-removed': repo.status == 'removed' || repo.status == 'expired'}"></span>
+      <span class="status-dot" ng-class="{'status-ready': repo.status == 'ready', 'status-error': repo.status == 'error', 'status-removed': repo.status == 'removed' || repo.status == 'expired', 'status-ratelimit': rateLimitResetAt}"></span>
-      <span ng-bind="repo.status | title"></span>
+      <span ng-if="!rateLimitResetAt" ng-bind="repo.status | title"></span>
+      <span ng-if="rateLimitResetAt">Rate limited</span>
     </span>
   </div>
 
@@ -19,7 +20,18 @@
       size. Visit the <a href="/faq">FAQ</a> for more information.
     </p>
 
-    <div class="paper-progress" ng-if="repo.status != 'error'" role="progressbar" aria-valuenow="{{progress}}" aria-valuemin="0" aria-valuemax="100" ng-class="{'paper-progress-ready': repo.status == 'ready'}">
+    <div class="paper-ratelimit-card" ng-if="rateLimitResetAt" role="status">
+      <div class="paper-ratelimit-head">
+        <i class="fas fa-hourglass-half"></i>
+        <div>
+          <div class="paper-error-eyebrow">Temporarily paused</div>
+          <div class="paper-error-title">GitHub API rate limit reached</div>
+        </div>
+      </div>
+      <p class="paper-error-msg">Anonymization will resume automatically in <strong>{{rateLimitCountdown}}</strong>. No action needed &mdash; the job is queued and will continue where it left off.</p>
+    </div>
+
+    <div class="paper-progress" ng-if="repo.status != 'error' && !rateLimitResetAt" role="progressbar" aria-valuenow="{{progress}}" aria-valuemin="0" aria-valuemax="100" ng-class="{'paper-progress-ready': repo.status == 'ready'}">
       <div class="paper-progress-bar" style="width: {{progress}}%;"></div>
       <div class="paper-progress-label">
         <span ng-bind="repo.status | title"></span><span ng-if="repo.statusMessage">&nbsp;&middot;&nbsp;<span ng-bind="repo.statusMessage"></span></span>
@@ -27,7 +39,7 @@
       </div>
     </div>
 
-    <div class="paper-error-card" ng-if="repo.status == 'error'" role="alert">
+    <div class="paper-error-card" ng-if="repo.status == 'error' && !rateLimitResetAt" role="alert">
       <div class="paper-error-head">
         <i class="fas fa-exclamation-triangle"></i>
         <div>

public/script/admin.js

@@ -919,12 +919,17 @@ angular
       $scope.selectedQueue = "download";
       $scope.selectedStats = null;
       $scope.range = "1h";
+      $scope.allStates = ["active", "waiting", "delayed", "failed", "completed"];
+      $scope.stateFilter = { active: true, waiting: true, delayed: true, failed: true, completed: true };
       $scope.query = {
         search: "",
-        state: "active",
         autoRefresh: true,
       };
 
+      $scope.filteredJobs = () => {
+        return ($scope.jobs || []).filter((j) => $scope.stateFilter[j._state]);
+      };
+
       $scope.jobProgressPct = (job) => {
         if (job && job.progress && typeof job.progress === "object" && typeof job.progress.percent === "number") {
           return Math.max(0, Math.min(100, Math.round(job.progress.percent)));
@@ -943,20 +948,22 @@ angular
         return (ms / 1000).toFixed(1) + "s";
       };
 
+      $scope.metricsPoints = [];
+
       $scope.selectQueue = (key) => {
         $scope.selectedQueue = key;
         getQueues();
+        getMetrics();
       };
 
       $scope.setRange = (r) => {
         $scope.range = r;
-        getQueues();
+        getMetrics();
       };
 
       function getQueues() {
         const params = {
           queue: $scope.selectedQueue,
-          state: $scope.query.state,
           search: $scope.query.search,
         };
         $http.get("/api/admin/queues", { params }).then(
@@ -964,26 +971,50 @@ angular
             $scope.queueList = res.data.queues || [];
             $scope.jobs = res.data.jobs || [];
             $scope.selectedStats = $scope.queueList.find((q) => q.key === $scope.selectedQueue) || $scope.queueList[0] || null;
+          },
+          (err) => console.error(err)
+        );
+      }
+
+      function getMetrics() {
+        $http.get("/api/admin/queues/metrics", {
+          params: { queue: $scope.selectedQueue, range: $scope.range }
+        }).then(
+          (res) => {
+            $scope.metricsPoints = res.data.points || [];
             $timeout(drawChart, 0);
           },
           (err) => console.error(err)
         );
       }
       getQueues();
+      getMetrics();
 
       const stop = $interval(() => {
-        if ($scope.query.autoRefresh) getQueues();
+        if ($scope.query.autoRefresh) {
+          getQueues();
+          getMetrics();
+        }
       }, 5000);
       $scope.$on("$destroy", () => $interval.cancel(stop));
 
-      $scope.refreshNow = getQueues;
+      $scope.refreshNow = function () { getQueues(); getMetrics(); };
+
+      function apiError(err) {
+        const msg = (err && err.data && (err.data.message || err.data.error)) || "Request failed";
+        $scope.actionError = msg;
+        $timeout(() => { $scope.actionError = null; }, 5000);
+        console.error(err);
+      }
+
+      $scope.actionError = null;
 
       $scope.removeJob = (job) => {
-        $http.delete(`/api/admin/queue/${$scope.selectedQueue}/${job.id}`).then(getQueues, (err) => console.error(err));
+        $http.delete(`/api/admin/queue/${$scope.selectedQueue}/${job.id}`).then(getQueues, apiError);
       };
 
       $scope.retryJob = (job) => {
-        $http.post(`/api/admin/queue/${$scope.selectedQueue}/${job.id}`).then(getQueues, (err) => console.error(err));
+        $http.post(`/api/admin/queue/${$scope.selectedQueue}/${job.id}`).then(getQueues, apiError);
       };
 
       $scope.retryFailed = () => {
@@ -1016,88 +1047,260 @@ angular
         clearTimeout(searchClear);
         searchClear = setTimeout(getQueues, 350);
       });
-      $scope.$watch("query.state", getQueues);
+      $scope.expanded = {};
+      $scope.toggleJob = (job) => {
+        $scope.expanded[job.id] = !$scope.expanded[job.id];
+      };
+
+      $scope.humanTime = (ts) => {
+        if (!ts) return "";
+        const d = new Date(ts);
+        return d.toLocaleTimeString([], { hour: "2-digit", minute: "2-digit", second: "2-digit", hour12: false })
+          + " " + d.toLocaleDateString([], { month: "short", day: "numeric" });
+      };
+
+      $scope.delayCountdown = (ts) => {
+        if (!ts) return "";
+        var remaining = Math.max(0, Math.ceil((ts - Date.now()) / 1000));
+        if (remaining <= 0) return "resuming soon";
+        var min = Math.floor(remaining / 60);
+        var sec = remaining % 60;
+        return "in " + (min > 0 ? min + "m " + sec + "s" : sec + "s");
+      };
+
+      function niceScale(max) {
+        if (max <= 0) return { ticks: [0], niceMax: 1 };
+        const mag = Math.pow(10, Math.floor(Math.log10(max)));
+        let step = mag;
+        if (max / step < 2) step = mag / 2;
+        else if (max / step > 5) step = mag * 2;
+        const niceMax = Math.ceil(max / step) * step;
+        const ticks = [];
+        for (let v = 0; v <= niceMax; v += step) ticks.push(v);
+        return { ticks, niceMax };
+      }
 
       function drawChart() {
-        const canvas = document.getElementById("q-throughput-chart");
+        var canvas = document.getElementById("q-throughput-chart");
-        if (!canvas || !$scope.selectedStats) return;
+        if (!canvas) return;
-        const ctx = canvas.getContext("2d");
+        var ctx = canvas.getContext("2d");
-        const dpr = window.devicePixelRatio || 1;
+        var dpr = window.devicePixelRatio || 1;
-        const rect = canvas.parentElement.getBoundingClientRect();
+        var rect = canvas.parentElement.getBoundingClientRect();
-        const w = rect.width - 40;
+        var marginLeft = 44;
-        const h = 160;
+        var marginRight = 50;
-        canvas.width = w * dpr;
+        var marginBottom = 20;
-        canvas.height = h * dpr;
+        var totalW = rect.width - 40;
-        canvas.style.width = w + "px";
+        var totalH = 180;
-        canvas.style.height = h + "px";
+        var w = totalW - marginLeft - marginRight;
+        var h = totalH - marginBottom;
+        canvas.width = totalW * dpr;
+        canvas.height = totalH * dpr;
+        canvas.style.width = totalW + "px";
+        canvas.style.height = totalH + "px";
         ctx.setTransform(dpr, 0, 0, dpr, 0, 0);
 
-        const data = ($scope.selectedStats.throughput || []).slice().reverse();
+        var isDark = document.body.classList.contains("dark-mode");
-        if (data.length === 0) {
+        var labelColor = "#8A857C";
-          ctx.fillStyle = getComputedStyle(document.documentElement).getPropertyValue("--ink-muted").trim() || "#8A857C";
+        var gridColor = isDark ? "rgba(255,255,255,0.06)" : "rgba(0,0,0,0.06)";
-          ctx.font = "12px var(--font-mono)";
+        var completedColor = isDark ? "#A7B2FF" : "#3B4AD6";
+        var completedFill = isDark ? "rgba(167,178,255,0.12)" : "rgba(59,74,214,0.08)";
+        var failedColor = isDark ? "#F08A82" : "#B42318";
+        var failedFill = isDark ? "rgba(240,138,130,0.08)" : "rgba(180,35,24,0.06)";
+        var execColor = isDark ? "#F5C842" : "#B8860B";
+
+        var pts = $scope.metricsPoints || [];
+        if (pts.length === 0) {
+          ctx.fillStyle = labelColor;
+          ctx.font = "12px monospace";
           ctx.textAlign = "center";
-          ctx.fillText("No throughput data yet", w / 2, h / 2);
+          ctx.fillText("No metrics data yet", totalW / 2, totalH / 2);
+          chartState = null;
           return;
         }
 
-        const rangePoints = { "1h": 60, "6h": 120, "24h": 120, "7d": 120 };
+        // Data is oldest→newest from the API; chart shows newest on the right
-        const pts = data.slice(0, rangePoints[$scope.range] || 60);
+        var completedPts = pts.map(function (p) { return p.completed; });
-        const max = Math.max(1, ...pts);
+        var failedPts = pts.map(function (p) { return p.failed; });
-        const step = w / (pts.length - 1 || 1);
+        var execPts = pts.map(function (p) { return p.avgMs; });
+        var maxLen = pts.length;
+        var step = w / (maxLen - 1 || 1);
 
-        const isDark = document.body.classList.contains("dark-mode");
+        // Left Y-axis: jobs/min
-        const lineColor = isDark ? "#A7B2FF" : "#3B4AD6";
+        var rawMax = Math.max(1, Math.max.apply(null, completedPts), Math.max.apply(null, failedPts));
-        const fillColor = isDark ? "rgba(167,178,255,0.12)" : "rgba(59,74,214,0.08)";
+        var left = niceScale(rawMax);
-        const gridColor = isDark ? "rgba(255,255,255,0.06)" : "rgba(0,0,0,0.06)";
 
-        // grid
+        // Right Y-axis: avg exec time (ms)
-        ctx.strokeStyle = gridColor;
+        var execMax = Math.max.apply(null, execPts);
-        ctx.lineWidth = 1;
+        var right = execMax > 0 ? niceScale(execMax) : { ticks: [0], niceMax: 1 };
-        for (let i = 0; i < 4; i++) {
+
-          const y = (h / 4) * i;
+        var toY = function (v) { return h - (v / left.niceMax) * (h - 10); };
+        var toYr = function (v) { return h - (v / right.niceMax) * (h - 10); };
+        var toX = function (i) { return marginLeft + i * step; };
+
+        // Grid + left Y-axis labels (jobs/min)
+        ctx.textAlign = "right";
+        ctx.textBaseline = "middle";
+        ctx.font = "10px monospace";
+        left.ticks.forEach(function (v) {
+          var y = toY(v);
+          ctx.strokeStyle = gridColor;
+          ctx.lineWidth = 1;
           ctx.beginPath();
-          ctx.moveTo(0, y);
+          ctx.moveTo(marginLeft, y);
-          ctx.lineTo(w, y);
+          ctx.lineTo(totalW - marginRight, y);
+          ctx.stroke();
+          ctx.fillStyle = labelColor;
+          ctx.fillText(v >= 1000 ? (v / 1000).toFixed(1) + "k" : String(v), marginLeft - 6, y);
+        });
+
+        // Right Y-axis labels (ms)
+        if (execMax > 0) {
+          ctx.textAlign = "left";
+          right.ticks.forEach(function (v) {
+            var y = toYr(v);
+            ctx.fillStyle = execColor;
+            ctx.fillText(v >= 1000 ? (v / 1000).toFixed(1) + "s" : v + "ms", totalW - marginRight + 6, y);
+          });
+        }
+
+        // X-axis time labels using actual timestamps
+        var now = Date.now();
+        var xLabelCount = Math.min(6, maxLen);
+        ctx.textAlign = "center";
+        ctx.textBaseline = "top";
+        for (var i = 0; i < xLabelCount; i++) {
+          var idx = Math.round((i / (xLabelCount - 1)) * (maxLen - 1));
+          var minsAgo = Math.round((now - pts[idx].ts) / 60000);
+          var x = toX(idx);
+          var lbl;
+          if (minsAgo <= 0) lbl = "now";
+          else if (minsAgo < 60) lbl = minsAgo + "m";
+          else if (minsAgo < 1440) lbl = Math.round(minsAgo / 60) + "h";
+          else lbl = Math.round(minsAgo / 1440) + "d";
+          ctx.fillStyle = labelColor;
+          ctx.fillText(lbl, x, h + 4);
+        }
+
+        function drawArea(data, yFn, fillStyle, strokeStyle) {
+          if (data.length === 0) return;
+          ctx.beginPath();
+          ctx.moveTo(toX(0), h);
+          data.forEach(function (v, i) {
+            var x = toX(i), y = yFn(v);
+            if (i === 0) ctx.lineTo(x, y);
+            else {
+              var cx = (toX(i - 1) + x) / 2;
+              ctx.bezierCurveTo(cx, yFn(data[i - 1]), cx, y, x, y);
+            }
+          });
+          ctx.lineTo(toX(data.length - 1), h);
+          ctx.closePath();
+          ctx.fillStyle = fillStyle;
+          ctx.fill();
+          ctx.beginPath();
+          data.forEach(function (v, i) {
+            var x = toX(i), y = yFn(v);
+            if (i === 0) ctx.moveTo(x, y);
+            else {
+              var cx = (toX(i - 1) + x) / 2;
+              ctx.bezierCurveTo(cx, yFn(data[i - 1]), cx, y, x, y);
+            }
+          });
+          ctx.strokeStyle = strokeStyle;
+          ctx.lineWidth = 1.5;
           ctx.stroke();
         }
 
-        // area fill
+        drawArea(completedPts, toY, completedFill, completedColor);
-        ctx.beginPath();
+        drawArea(failedPts, toY, failedFill, failedColor);
-        ctx.moveTo(0, h);
+        // Exec time as a line only (no fill) on the right axis
-        pts.forEach((v, i) => {
+        if (execMax > 0) {
-          const x = i * step;
+          ctx.beginPath();
-          const y = h - (v / max) * (h - 10);
+          execPts.forEach(function (v, i) {
-          if (i === 0) ctx.lineTo(x, y);
+            var x = toX(i), y = toYr(v);
-          else {
+            if (i === 0) ctx.moveTo(x, y);
-            const px = (i - 1) * step;
+            else {
-            const py = h - (pts[i - 1] / max) * (h - 10);
+              var cx = (toX(i - 1) + x) / 2;
-            const cx = (px + x) / 2;
+              ctx.bezierCurveTo(cx, toYr(execPts[i - 1]), cx, y, x, y);
-            ctx.bezierCurveTo(cx, py, cx, y, x, y);
+            }
-          }
+          });
-        });
+          ctx.strokeStyle = execColor;
-        ctx.lineTo(w, h);
+          ctx.lineWidth = 1;
-        ctx.closePath();
+          ctx.setLineDash([4, 3]);
-        ctx.fillStyle = fillColor;
+          ctx.stroke();
-        ctx.fill();
+          ctx.setLineDash([]);
+        }
 
-        // line
+        chartState = { pts: pts, maxLen: maxLen, marginLeft: marginLeft, step: step, totalW: totalW, toX: toX };
-        ctx.beginPath();
-        pts.forEach((v, i) => {
-          const x = i * step;
-          const y = h - (v / max) * (h - 10);
-          if (i === 0) ctx.moveTo(x, y);
-          else {
-            const px = (i - 1) * step;
-            const py = h - (pts[i - 1] / max) * (h - 10);
-            const cx = (px + x) / 2;
-            ctx.bezierCurveTo(cx, py, cx, y, x, y);
-          }
-        });
-        ctx.strokeStyle = lineColor;
-        ctx.lineWidth = 1.5;
-        ctx.stroke();
       }
+
+      var chartState = null;
+
+      function setupTooltip() {
+        var canvas = document.getElementById("q-throughput-chart");
+        if (!canvas || canvas._tipBound) return;
+        canvas._tipBound = true;
+
+        var tooltip = document.getElementById("q-chart-tooltip");
+        var crosshair = document.getElementById("q-chart-crosshair");
+
+        canvas.addEventListener("mousemove", function (e) {
+          if (!chartState || !tooltip || !crosshair) return;
+          var cs = chartState;
+          var rect = canvas.getBoundingClientRect();
+          var mx = e.clientX - rect.left;
+
+          var idx = Math.round((mx - cs.marginLeft) / cs.step);
+          if (idx < 0 || idx >= cs.maxLen) {
+            tooltip.style.display = "none";
+            crosshair.style.display = "none";
+            return;
+          }
+
+          var p = cs.pts[idx];
+          var now = Date.now();
+          var minsAgo = Math.round((now - p.ts) / 60000);
+          var timeLabel;
+          if (minsAgo <= 0) timeLabel = "now";
+          else if (minsAgo < 60) timeLabel = minsAgo + "m ago";
+          else if (minsAgo < 1440) {
+            var hrs = Math.floor(minsAgo / 60);
+            var mins = minsAgo % 60;
+            timeLabel = hrs + "h" + (mins ? " " + mins + "m" : "") + " ago";
+          } else timeLabel = Math.round(minsAgo / 1440) + "d ago";
+
+          var html =
+            '<div class="q-tip-time">' + timeLabel + '</div>' +
+            '<div class="q-tip-completed">&#9679; completed: ' + p.completed + '/min</div>' +
+            '<div class="q-tip-failed">&#9679; failed: ' + p.failed + '/min</div>';
+          if (p.avgMs > 0) {
+            var dur = p.avgMs >= 1000 ? (p.avgMs / 1000).toFixed(1) + "s" : p.avgMs + "ms";
+            html += '<div class="q-tip-exec">&#9679; avg time: ' + dur + '</div>';
+          }
+          tooltip.innerHTML = html;
+
+          var xPos = cs.toX(idx);
+          var tipW = tooltip.offsetWidth;
+          var tipLeft = xPos + 10;
+          if (tipLeft + tipW > cs.totalW) tipLeft = xPos - tipW - 10;
+
+          tooltip.style.display = "block";
+          tooltip.style.left = tipLeft + "px";
+          tooltip.style.top = "8px";
+
+          crosshair.style.display = "block";
+          crosshair.style.left = xPos + "px";
+        });
+
+        canvas.addEventListener("mouseleave", function () {
+          if (tooltip) tooltip.style.display = "none";
+          if (crosshair) crosshair.style.display = "none";
+        });
+      }
+
+      $scope.$watch("metricsPoints", function () {
+        $timeout(setupTooltip, 50);
+      });
     },
   ])
   .controller("errorsAdminController", [

public/script/app.js

@@ -226,6 +226,20 @@ angular
       return capitalized.join(" ");
     };
   })
+  .filter("statusMsg", function () {
+    return function (msg) {
+      if (!msg) return msg;
+      var m = msg.match(/^rate_limited:(\d+)$/);
+      if (m) {
+        var remaining = Math.max(0, Math.ceil((parseInt(m[1], 10) - Date.now()) / 1000));
+        if (remaining <= 0) return "Rate limited — resuming soon";
+        var min = Math.floor(remaining / 60);
+        var sec = remaining % 60;
+        return "Rate limited — retrying in " + (min > 0 ? min + "m " + sec + "s" : sec + "s");
+      }
+      return msg;
+    };
+  })
   .filter("diff", [
     "$sce",
     function ($sce) {
@@ -1520,6 +1534,47 @@ angular
       $scope.repoId = $routeParams.repoId;
       $scope.repo = null;
       $scope.progress = 0;
+      $scope.rateLimitResetAt = 0;
+      $scope.rateLimitCountdown = "";
+
+      var countdownTimer = null;
+      function startRateLimitCountdown(resetAt) {
+        $scope.rateLimitResetAt = resetAt;
+        if (countdownTimer) clearInterval(countdownTimer);
+        function tick() {
+          var remaining = Math.max(0, Math.ceil((resetAt - Date.now()) / 1000));
+          if (remaining <= 0) {
+            $scope.rateLimitCountdown = "";
+            $scope.rateLimitResetAt = 0;
+            clearInterval(countdownTimer);
+            countdownTimer = null;
+          } else {
+            var min = Math.floor(remaining / 60);
+            var sec = remaining % 60;
+            $scope.rateLimitCountdown = min > 0
+              ? min + "m " + sec + "s"
+              : sec + "s";
+          }
+          $scope.$applyAsync();
+        }
+        tick();
+        countdownTimer = setInterval(tick, 1000);
+      }
+      $scope.$on("$destroy", function () {
+        if (countdownTimer) clearInterval(countdownTimer);
+      });
+
+      function parseStatusMessage(msg) {
+        if (!msg) return msg;
+        var m = msg.match(/^rate_limited:(\d+)$/);
+        if (m) {
+          startRateLimitCountdown(parseInt(m[1], 10));
+          return null;
+        }
+        $scope.rateLimitResetAt = 0;
+        return msg;
+      }
+
       $scope.getStatus = () => {
         $http
           .get("/api/repo/" + $scope.repoId, {
@@ -1529,6 +1584,11 @@ angular
           .then(
             (res) => {
               $scope.repo = res.data;
+              if (res.data.rateLimitResetAt) {
+                startRateLimitCountdown(res.data.rateLimitResetAt);
+              } else {
+                $scope.repo.statusMessage = parseStatusMessage($scope.repo.statusMessage);
+              }
               if ($scope.repo.status == "ready") {
                 $scope.progress = 100;
               } else if ($scope.repo.status == "queue") {
@@ -1542,10 +1602,11 @@ angular
               } else if ($scope.repo.status == "anonymizing") {
                 $scope.progress = 75;
               }
-              if (
+              var shouldPoll = $scope.repo.status != "ready";
-                $scope.repo.status != "ready" &&
+              if ($scope.repo.status == "error" && !$scope.rateLimitResetAt) {
-                $scope.repo.status != "error"
+                shouldPoll = false;
-              ) {
+              }
+              if (shouldPoll) {
                 setTimeout($scope.getStatus, 2000);
               }
             },
@@ -2598,12 +2659,14 @@ angular
         )[0];
       }
 
+      var rlCountdownTimer = null;
+      $scope.$on("$destroy", function () { if (rlCountdownTimer) clearInterval(rlCountdownTimer); });
+
       function getOptions(callback) {
         $http.get(`/api/repo/${$scope.repoId}/options`).then(
           (res) => {
             $scope.options = res.data;
             if ($scope.options.url) {
-              // the repository is expired with redirect option
               window.location = $scope.options.url;
               return;
             }
@@ -2612,8 +2675,34 @@ angular
             }
           },
           (err) => {
-            $scope.type = "error";
+            var data = err.data || {};
-            $scope.content = err.data.error;
+            if (data.error === "rate_limited" && data.resetAt) {
+              $scope.type = "rate_limited";
+              $scope.rateLimitResetAt = data.resetAt;
+              if (rlCountdownTimer) clearInterval(rlCountdownTimer);
+              function rlTick() {
+                var remaining = Math.max(0, Math.ceil(($scope.rateLimitResetAt - Date.now()) / 1000));
+                if (remaining <= 0) {
+                  $scope.rateLimitCountdown = "";
+                  $scope.rateLimitResetAt = 0;
+                  if (rlCountdownTimer) { clearInterval(rlCountdownTimer); rlCountdownTimer = null; }
+                  getOptions(callback);
+                } else {
+                  var min = Math.floor(remaining / 60);
+                  var sec = remaining % 60;
+                  $scope.rateLimitCountdown = min > 0 ? min + "m " + sec + "s" : sec + "s";
+                }
+                $scope.$applyAsync();
+              }
+              rlTick();
+              rlCountdownTimer = setInterval(rlTick, 1000);
+            } else if (data.error === "repository_not_ready") {
+              $scope.type = "loading";
+              setTimeout(function () { getOptions(callback); }, 3000);
+            } else {
+              $scope.type = "error";
+              $scope.content = data.error;
+            }
           }
         );
       }

src/core/GitHubUtils.ts

@@ -1,5 +1,6 @@
 import { Octokit } from "@octokit/rest";
 import { throttling } from "@octokit/plugin-throttling";
+import { createClient, RedisClientType } from "redis";
 
 import AnonymousError from "./AnonymousError";
 import Repository from "./Repository";
@@ -53,6 +54,155 @@ function rateLimitDetail(err: OctokitRequestErrorLike): string {
 
 const ThrottledOctokit = Octokit.plugin(throttling);
 
+/**
+ * Per-token gate that blocks all callers when a rate limit is active.
+ * When any request for a given token hits a rate limit, the gate records
+ * the reset time and makes every subsequent caller wait until then —
+ * preventing a stampede of doomed requests.
+ */
+const tokenGates = new Map<string, { resetAt: number }>();
+
+function setTokenGate(token: string, retryAfterSec: number) {
+  const key = token.slice(-8);
+  const resetAt = Date.now() + retryAfterSec * 1000;
+  const existing = tokenGates.get(key);
+  if (!existing || resetAt > existing.resetAt) {
+    tokenGates.set(key, { resetAt });
+    logger.warn("rate limit gate set", {
+      code: "rate_limit_gate",
+      retryAfterSec,
+      resetAt: new Date(resetAt).toISOString(),
+    });
+    setRedisGate(retryAfterSec).catch(() => {});
+  }
+}
+
+export class RateLimitDelayError extends Error {
+  resetAt: number;
+  constructor(resetAt: number) {
+    const delaySec = Math.ceil((resetAt - Date.now()) / 1000);
+    super(`github_rate_limit_delay:${delaySec}s`);
+    this.name = "RateLimitDelayError";
+    this.resetAt = resetAt;
+  }
+}
+
+/**
+ * Check if a rate limit gate is active for a token.
+ * Returns the reset timestamp, or 0 if no gate is active.
+ */
+export function getTokenGateResetAt(token: string): number {
+  const key = token.slice(-8);
+  const gate = tokenGates.get(key);
+  if (!gate) return 0;
+  if (gate.resetAt <= Date.now()) {
+    tokenGates.delete(key);
+    return 0;
+  }
+  return gate.resetAt;
+}
+
+async function waitForTokenGate(token: string): Promise<void> {
+  const key = token.slice(-8);
+  const localGate = tokenGates.get(key);
+  let waitMs = 0;
+  let resetAt = 0;
+
+  if (localGate && localGate.resetAt > Date.now()) {
+    resetAt = localGate.resetAt;
+    waitMs = resetAt - Date.now();
+  }
+
+  const redisResetAt = await getRedisGateResetAt();
+  if (redisResetAt > resetAt) {
+    resetAt = redisResetAt;
+    waitMs = resetAt - Date.now();
+  }
+
+  if (waitMs <= 0) {
+    if (localGate) tokenGates.delete(key);
+    return;
+  }
+
+  logger.info("waiting for rate limit gate", {
+    code: "rate_limit_gate_wait",
+    waitMs,
+    resetAt: new Date(resetAt).toISOString(),
+  });
+  await new Promise((resolve) => setTimeout(resolve, waitMs));
+  if (localGate) tokenGates.delete(key);
+}
+
+const REDIS_GATE_PREFIX = "gh_rate_gate:";
+
+let redisGateDisabled = false;
+let redisGateReady: Promise<RedisClientType | null> | null = null;
+
+function ensureRedisGateClient(): Promise<RedisClientType | null> {
+  if (redisGateDisabled) return Promise.resolve(null);
+  if (redisGateReady) return redisGateReady;
+  redisGateReady = (async () => {
+    try {
+      const c = createClient({
+        socket: {
+          host: config.REDIS_HOSTNAME,
+          port: config.REDIS_PORT,
+          reconnectStrategy: () => false as any,
+        },
+      }) as RedisClientType;
+      c.on("error", () => {
+        redisGateDisabled = true;
+        c.disconnect().catch(() => {});
+        redisGateReady = null;
+      });
+      await c.connect();
+      return c;
+    } catch {
+      redisGateDisabled = true;
+      redisGateReady = null;
+      return null;
+    }
+  })();
+  return redisGateReady;
+}
+
+async function setRedisGate(retryAfterSec: number): Promise<void> {
+  const c = await ensureRedisGateClient();
+  if (!c || !c.isOpen) return;
+  const resetAt = Date.now() + retryAfterSec * 1000;
+  const ttl = Math.ceil(retryAfterSec) + 10;
+  try {
+    await c.set(REDIS_GATE_PREFIX + "global", String(resetAt), { EX: ttl });
+    logger.info("redis rate limit gate written", {
+      code: "redis_gate_set",
+      resetAt: new Date(resetAt).toISOString(),
+      ttl,
+    });
+  } catch {
+    // non-critical
+  }
+}
+
+export async function setRedisGateFromWorker(resetAt: number): Promise<void> {
+  const retryAfterSec = Math.max(0, (resetAt - Date.now()) / 1000);
+  if (retryAfterSec <= 0) return;
+  await setRedisGate(retryAfterSec);
+}
+
+export async function getRedisGateResetAt(): Promise<number> {
+  const c = await ensureRedisGateClient();
+  if (!c || !c.isOpen) return 0;
+  try {
+    const val = await c.get(REDIS_GATE_PREFIX + "global");
+    if (!val) return 0;
+    const resetAt = parseInt(val, 10);
+    if (isNaN(resetAt) || resetAt <= Date.now()) return 0;
+    return resetAt;
+  } catch {
+    return 0;
+  }
+}
+
 export function octokit(token: string) {
   const oct = new ThrottledOctokit({
     auth: token,
@@ -69,8 +219,7 @@ export function octokit(token: string) {
           retryAfter,
           retryCount,
         });
-        // Retry once; if GitHub is still throttling after that, surface the
+        setTokenGate(token, retryAfter);
-        // error to the caller so the UI shows github_rate_limit_exceeded.
         return retryCount < 1;
       },
       onSecondaryRateLimit: (retryAfter, options, _o, retryCount) => {
@@ -82,6 +231,7 @@ export function octokit(token: string) {
           retryAfter,
           retryCount,
         });
+        setTokenGate(token, retryAfter);
         return retryCount < 1;
       },
     },
@@ -99,12 +249,20 @@ export function octokit(token: string) {
   return oct;
 }
 
+export { waitForTokenGate };
+
 export async function checkToken(token: string) {
   const oct = octokit(token);
   try {
     await oct.users.getAuthenticated();
     return true;
-  } catch {
+  } catch (err) {
+    if (
+      err instanceof AnonymousError &&
+      err.message === "github_rate_limit_exceeded"
+    ) {
+      throw err;
+    }
     return false;
   }
 }

src/core/Repository.ts

@@ -19,7 +19,7 @@ import {
   getRepositoryFromGitHub,
   GitHubRepository,
 } from "./source/GitHubRepository";
-import { getToken } from "./GitHubUtils";
+import { getToken, getRedisGateResetAt } from "./GitHubUtils";
 import config from "../config";
 import FileModel from "./model/files/files.model";
 import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
@@ -234,14 +234,30 @@ export default class Repository {
         httpStatus: 410,
       });
     }
+    const redisGateReset = await getRedisGateResetAt();
+    if (redisGateReset > 0) {
+      throw new AnonymousError("rate_limited", {
+        httpStatus: 425,
+        object: { resetAt: redisGateReset },
+      });
+    }
+
     const fiveMinuteAgo = new Date();
     fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
 
     if (
       this.status == RepositoryStatus.PREPARING ||
+      this.status == RepositoryStatus.QUEUE ||
       (this.status == RepositoryStatus.DOWNLOAD &&
         this._model.statusDate > fiveMinuteAgo)
     ) {
+      const rlMatch = (this._model.statusMessage || "").match(/^rate_limited:(\d+)$/);
+      if (rlMatch) {
+        throw new AnonymousError("rate_limited", {
+          httpStatus: 425,
+          object: { resetAt: parseInt(rlMatch[1], 10) },
+        });
+      }
       throw new AnonymousError("repository_not_ready", {
         object: this,
         httpStatus: 425,

src/core/source/GitHubStream.ts

@@ -11,7 +11,7 @@ import { basename, dirname } from "path";
 import * as stream from "stream";
 import AnonymousError from "../AnonymousError";
 import { FILE_TYPE } from "../storage/Storage";
-import { octokit } from "../GitHubUtils";
+import { octokit, waitForTokenGate } from "../GitHubUtils";
 import FileModel from "../model/files/files.model";
 import { IFile } from "../model/files/files.types";
 import { createLogger, serializeError } from "../logger";
@@ -20,6 +20,27 @@ import config from "../../config";
 
 const logger = createLogger("gh-stream");
 
+const GH_API_CONCURRENCY = 6;
+
+async function pMap<T, R>(
+  items: T[],
+  fn: (item: T, index: number) => Promise<R>,
+  concurrency: number
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let next = 0;
+  async function worker() {
+    while (next < items.length) {
+      const i = next++;
+      results[i] = await fn(items[i], i);
+    }
+  }
+  await Promise.all(
+    Array.from({ length: Math.min(concurrency, items.length) }, () => worker())
+  );
+  return results;
+}
+
 export function githubRawFileUrl(
   owner: string,
   repo: string,
@@ -354,11 +375,13 @@ export default class GitHubStream extends GitHubBase {
   }
 
   private async getGHTree(
+    oct: ReturnType<typeof octokit>,
+    token: string,
     sha: string,
     count = { request: 0, file: 0 },
     opt = { recursive: true, callback: () => {} }
   ) {
-    const oct = octokit(await this.data.getToken());
+    await waitForTokenGate(token);
     const ghRes = await oct.git.getTree({
       owner: this.data.organization,
       repo: this.data.repoName,
@@ -378,6 +401,8 @@ export default class GitHubStream extends GitHubBase {
     progress?: (status: string) => void,
     parentPath: string = ""
   ) {
+    const token = await this.data.getToken();
+    const oct = octokit(token);
     const count = {
       request: 0,
       file: 0,
@@ -385,7 +410,7 @@ export default class GitHubStream extends GitHubBase {
     const output: IFile[] = [];
     let data;
     try {
-      data = await this.getGHTree(sha, count, {
+      data = await this.getGHTree(oct, token, sha, count, {
         recursive: false,
         callback: () => {
           if (progress) {
@@ -423,29 +448,33 @@ export default class GitHubStream extends GitHubBase {
         cause: error as Error,
       });
     }
-    const promises: ReturnType<GitHubStream["getGHTree"]>[] = [];
+    const subtrees: { sha: string; parentPath: string }[] = [];
-    const parentPaths: string[] = [];
     for (const file of data.tree) {
       if (file.type == "tree" && file.path && file.sha) {
-        const elementPath = path.join(parentPath, file.path);
+        subtrees.push({
-        parentPaths.push(elementPath);
+          sha: file.sha,
-        promises.push(
+          parentPath: path.join(parentPath, file.path),
-          this.getGHTree(file.sha, count, {
+        });
-            recursive: true,
-            callback: () => {
-              if (progress) {
-                progress("List file: " + count.file);
-              }
-            },
-          })
-        );
       }
     }
-    (await Promise.all(promises)).forEach((data, i) => {
+    const results = await pMap(
+      subtrees,
+      async (entry) =>
+        this.getGHTree(oct, token, entry.sha, count, {
+          recursive: true,
+          callback: () => {
+            if (progress) {
+              progress("List file: " + count.file);
+            }
+          },
+        }),
+      GH_API_CONCURRENCY
+    );
+    results.forEach((data, i) => {
       if (data.truncated) {
-        this._truncatedFolders.push(parentPaths[i]);
+        this._truncatedFolders.push(subtrees[i].parentPath);
       }
-      output.push(...this.tree2Tree(data.tree, parentPaths[i]));
+      output.push(...this.tree2Tree(data.tree, subtrees[i].parentPath));
     });
     return output;
   }

src/queue/index.ts

@@ -4,6 +4,7 @@ import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anon
 import { RepositoryStatus } from "../core/types";
 import * as path from "path";
 import { createLogger, serializeError } from "../core/logger";
+import { recordMetric } from "./queueMetrics";
 
 const logger = createLogger("queue");
 
@@ -119,12 +120,15 @@ export function startWorker() {
       concurrency: 5,
       connection,
       autorun: true,
-      metrics: { maxDataPoints: 120 },
     }
   );
   cacheWorker.on("completed", async (job) => {
+    recordMetric("cache", "completed", (job.finishedOn || Date.now()) - (job.processedOn || job.timestamp));
     await job.remove();
   });
+  cacheWorker.on("failed", async (job) => {
+    if (job) recordMetric("cache", "failed", Date.now() - (job.processedOn || job.timestamp));
+  });
   const removeWorker = new Worker<RepoJobData>(
     removeQueue.name,
     path.resolve("build/queue/processes/removeRepository.js"),
@@ -132,12 +136,15 @@ export function startWorker() {
       concurrency: 5,
       connection,
       autorun: true,
-      metrics: { maxDataPoints: 120 },
     }
   );
   removeWorker.on("completed", async (job) => {
+    recordMetric("remove", "completed", (job.finishedOn || Date.now()) - (job.processedOn || job.timestamp));
     await job.remove();
   });
+  removeWorker.on("failed", async (job) => {
+    if (job) recordMetric("remove", "failed", Date.now() - (job.processedOn || job.timestamp));
+  });
 
   const downloadWorker = new Worker<RepoJobData>(
     downloadQueue.name,
@@ -146,7 +153,6 @@ export function startWorker() {
       concurrency: 3,
       connection,
       autorun: true,
-      metrics: { maxDataPoints: 120 },
     }
   );
   if (!downloadWorker.isRunning()) downloadWorker.run();
@@ -156,8 +162,10 @@ export function startWorker() {
   });
   downloadWorker.on("completed", async (job) => {
     logger.info("download completed", { repoId: job.data.repoId });
+    recordMetric("download", "completed", (job.finishedOn || Date.now()) - (job.processedOn || job.timestamp));
   });
   downloadWorker.on("failed", async (job, err) => {
+    if (job) recordMetric("download", "failed", Date.now() - (job.processedOn || job.timestamp));
     const repoId = job?.data?.repoId;
     logger.error("download failed", {
       ...serializeError(err),

src/queue/processes/downloadRepository.ts

@@ -5,6 +5,8 @@ import { getRepository as getRepositoryImport } from "../../server/database";
 import { RepositoryStatus } from "../../core/types";
 import { RepoJobData } from "../index";
 import { createLogger, serializeError } from "../../core/logger";
+import { RateLimitDelayError, getRedisGateResetAt, setRedisGateFromWorker } from "../../core/GitHubUtils";
+import { DelayedError } from "bullmq";
 
 const logger = createLogger("queue:download");
 
@@ -20,6 +22,24 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   let statusInterval: any = null;
   await connect();
+
+  const gateResetAt = await getRedisGateResetAt();
+  if (gateResetAt > 0) {
+    const delaySec = Math.ceil((gateResetAt - Date.now()) / 1000);
+    logger.info("rate limit gate active, delaying job before work", {
+      repoId: job.data.repoId,
+      delaySec,
+      resetAt: new Date(gateResetAt).toISOString(),
+    });
+    const repo = await getRepository(job.data.repoId);
+    await repo.updateStatus(
+      RepositoryStatus.QUEUE,
+      `rate_limited:${gateResetAt}`
+    );
+    await job.moveToDelayed(gateResetAt);
+    throw new DelayedError();
+  }
+
   const repo = await getRepository(job.data.repoId);
   let tickPromise: Promise<void> | null = null;
   try {
@@ -68,6 +88,30 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
     } catch (error) {
       clearInterval(statusInterval);
       if (tickPromise) await tickPromise;
+
+      // Rate-limited: delay the job and free the worker slot
+      const isRateDelay = error instanceof RateLimitDelayError;
+      const isRateError = !isRateDelay && error instanceof Error &&
+        (error.message === "github_rate_limit_exceeded" || error.message.includes("rate limit"));
+      if (isRateDelay || isRateError) {
+        const resetAt = isRateDelay
+          ? (error as RateLimitDelayError).resetAt
+          : Date.now() + 60_000; // fallback: retry in 1 min
+        const delaySec = Math.ceil(Math.max(0, resetAt - Date.now()) / 1000);
+        logger.info("rate-limited, delaying job", {
+          repoId: job.data.repoId,
+          delaySec,
+          resetAt: new Date(resetAt).toISOString(),
+        });
+        await setRedisGateFromWorker(resetAt);
+        await repo.updateStatus(
+          RepositoryStatus.QUEUE,
+          `rate_limited:${resetAt}`
+        );
+        await job.moveToDelayed(resetAt);
+        throw new DelayedError();
+      }
+
       updateProgress({ status: "error" });
       if (error instanceof Error) {
         await repo.updateStatus(RepositoryStatus.ERROR, error.message);
@@ -83,6 +127,9 @@ export default async function (job: SandboxedJob<RepoJobData, void>) {
         await tickPromise;
       } catch { /* ignored */ }
     }
+    if (error instanceof DelayedError || (error instanceof Error && error.name === "DelayedError")) {
+      throw error;
+    }
     logger.error("finished with error", {
       ...serializeError(error),
       repoId: job.data.repoId,

src/queue/queueMetrics.ts

@@ -0,0 +1,110 @@
+import { createClient, RedisClientType } from "redis";
+import config from "../config";
+
+const KEY_PREFIX = "qmetrics";
+const TTL_SECONDS = 7 * 24 * 3600 + 3600;
+
+let client: RedisClientType | null = null;
+let disabled = false;
+
+function getClient(): RedisClientType | null {
+  if (disabled) return null;
+  if (client) return client;
+  try {
+    client = createClient({
+      socket: {
+        host: config.REDIS_HOSTNAME,
+        port: config.REDIS_PORT,
+        reconnectStrategy: () => false as any,
+      },
+    }) as RedisClientType;
+    client.on("error", () => {
+      disabled = true;
+      client?.disconnect().catch(() => {});
+      client = null;
+    });
+    client.connect().catch(() => {
+      disabled = true;
+      client = null;
+    });
+    return client;
+  } catch {
+    disabled = true;
+    return null;
+  }
+}
+
+function minuteTs(now?: number): number {
+  return Math.floor((now || Date.now()) / 60000) * 60000;
+}
+
+function key(queue: string, ts: number): string {
+  return `${KEY_PREFIX}:${queue}:${ts}`;
+}
+
+export async function recordMetric(
+  queue: string,
+  type: "completed" | "failed",
+  durationMs: number
+): Promise<void> {
+  const c = getClient();
+  if (!c || !c.isOpen) return;
+  const k = key(queue, minuteTs());
+  const field = type === "completed" ? "c" : "f";
+  try {
+    const pipe = c.multi();
+    pipe.hIncrBy(k, field, 1);
+    pipe.hIncrBy(k, "ms", Math.round(Math.max(0, durationMs)));
+    pipe.expire(k, TTL_SECONDS);
+    await pipe.exec();
+  } catch {
+    // non-critical, don't crash workers
+  }
+}
+
+export interface MetricPoint {
+  ts: number;
+  completed: number;
+  failed: number;
+  avgMs: number;
+}
+
+export async function queryMetrics(
+  queue: string,
+  rangeMinutes: number
+): Promise<MetricPoint[]> {
+  const c = getClient();
+  if (!c || !c.isOpen) return [];
+
+  const now = minuteTs();
+  const start = now - (rangeMinutes - 1) * 60000;
+
+  const keys: string[] = [];
+  const timestamps: number[] = [];
+  for (let t = start; t <= now; t += 60000) {
+    keys.push(key(queue, t));
+    timestamps.push(t);
+  }
+
+  try {
+    const pipe = c.multi();
+    for (const k of keys) pipe.hGetAll(k);
+    const results = await pipe.exec();
+
+    return timestamps.map((ts, i) => {
+      const h = (results[i] as unknown as Record<string, string>) || {};
+      const completed = parseInt(h.c || "0", 10) || 0;
+      const failed = parseInt(h.f || "0", 10) || 0;
+      const totalMs = parseInt(h.ms || "0", 10) || 0;
+      const total = completed + failed;
+      return {
+        ts,
+        completed,
+        failed,
+        avgMs: total > 0 ? Math.round(totalMs / total) : 0,
+      };
+    });
+  } catch {
+    return [];
+  }
+}

src/server/routes/admin.ts

@@ -7,6 +7,7 @@ import AnonymizedRepositoryModel from "../../core/model/anonymizedRepositories/a
 import ConferenceModel from "../../core/model/conference/conferences.model";
 import UserModel from "../../core/model/users/users.model";
 import { cacheQueue, downloadQueue, removeQueue } from "../../queue";
+import { queryMetrics } from "../../queue/queueMetrics";
 import User from "../../core/User";
 import { ensureAuthenticated } from "./connection";
 import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
@@ -131,25 +132,25 @@ function sendCsv(
 router.post("/queue/:name/:repo_id", async (req, res) => {
   const queue = pickQueue(req.params.name);
   if (!queue) return res.status(404).json({ error: "queue_not_found" });
-  let job;
   try {
-    job = await queue.getJob(req.params.repo_id);
+    const job = await queue.getJob(req.params.repo_id);
     if (!job) {
       return res.status(404).json({ error: "job_not_found" });
     }
-
+    const state = await job.getState();
-    await job.retry();
+    if (state === "active") {
-    res.send("ok");
+      return res.status(409).json({ error: "job_is_active", message: "Cannot retry an active job — wait for it to finish or remove it first." });
-  } catch {
-    try {
-      if (job) {
-        await job.remove();
-        queue.add(job.name, job.data, job.opts);
-      }
-      res.send("ok");
-    } catch {
-      res.status(500).json({ error: "error_retrying_job" });
     }
+    try {
+      await job.retry();
+    } catch {
+      const { name, data, opts } = job;
+      await job.remove().catch(() => {});
+      await queue.add(name, data, opts);
+    }
+    res.json({ ok: true });
+  } catch (error) {
+    handleError(error, res, req);
   }
 });
 
@@ -157,12 +158,24 @@ router.delete("/queue/:name/:repo_id", async (req, res) => {
   const queue = pickQueue(req.params.name);
   if (!queue) return res.status(404).json({ error: "queue_not_found" });
   try {
-    const job = await queue.getJob(req.params.repo_id);
+    const jobId = req.params.repo_id;
+    const job = await queue.getJob(jobId);
     if (!job) {
       return res.status(404).json({ error: "job_not_found" });
     }
+
+    const state = await job.getState();
+
+    if (state === "active") {
+      // Active jobs hold a worker lock — delete it so remove() succeeds
+      const client = await (queue as any).client;
+      const lockKey = queue.toKey(jobId) + ":lock";
+      await client.del(lockKey);
+      logger.info("cleared lock for active job", { queue: queue.name, jobId });
+    }
+
     await job.remove();
-    res.send("ok");
+    res.json({ ok: true });
   } catch (error) {
     handleError(error, res, req);
   }
@@ -243,37 +256,58 @@ router.post("/queues/pause-all", async (_req, res) => {
   }
 });
 
-async function queueStats(queue: Queue) {
+async function queueStats(queueKey: string, queue: Queue) {
-  const [counts, workers, paused, completedMetrics, failedMetrics] =
+  const [counts, workers, paused, metrics24h] =
     await Promise.all([
       queue.getJobCounts(...QUEUE_STATES),
       queue.getWorkers().catch(() => []),
       queue.isPaused().catch(() => false),
-      queue.getMetrics("completed", 0, 119).catch(() => ({ data: [], count: 0 })),
+      queryMetrics(queueKey, 1440),
-      queue.getMetrics("failed", 0, 119).catch(() => ({ data: [], count: 0 })),
     ]);
 
   const workerCount = workers.length;
   const concurrency = workerCount > 0 ? (workers as any)[0]?.opts?.concurrency ?? null : null;
 
+  let completed24h = 0;
+  let failed24h = 0;
+  for (const p of metrics24h) {
+    completed24h += p.completed;
+    failed24h += p.failed;
+  }
+
   return {
     counts,
     paused,
     workers: workerCount,
     concurrency,
-    throughput: completedMetrics.data || [],
+    completed24h,
-    completed24h: completedMetrics.count || 0,
+    failed24h,
-    failed24h: failedMetrics.count || 0,
   };
 }
 
+const RANGE_MINUTES: Record<string, number> = {
+  "1h": 60,
+  "6h": 360,
+  "24h": 1440,
+  "7d": 10080,
+};
+
+router.get("/queues/metrics", async (req, res) => {
+  const queueName = String(req.query.queue || "download");
+  if (!pickQueue(queueName)) return res.status(404).json({ error: "queue_not_found" });
+  const range = String(req.query.range || "1h");
+  const minutes = RANGE_MINUTES[range] || 60;
+  try {
+    const points = await queryMetrics(queueName, minutes);
+    res.json({ queue: queueName, range, points });
+  } catch (error) {
+    handleError(error, res, req);
+  }
+});
+
 router.get("/queues", async (req, res) => {
   const search = req.query.search ? String(req.query.search).toLowerCase() : "";
   const queueName = req.query.queue ? String(req.query.queue) : "";
-  const stateFilter: JobType | null = req.query.state ? String(req.query.state) as JobType : null;
-  const states: JobType[] = stateFilter && QUEUE_STATES.includes(stateFilter)
-    ? [stateFilter]
-    : QUEUE_STATES;
 
   const allQueues: { key: string; label: string; queue: Queue }[] = [
     { key: "download", label: "Download", queue: downloadQueue },
@@ -285,7 +319,7 @@ router.get("/queues", async (req, res) => {
     allQueues.map(async (q) => ({
       key: q.key,
       label: q.label,
-      ...(await queueStats(q.queue)),
+      ...(await queueStats(q.key, q.queue)),
     }))
   );
 
@@ -294,8 +328,6 @@ router.get("/queues", async (req, res) => {
     : allQueues[0];
   const targetQueue = target ? target.queue : downloadQueue;
 
-  const jobs = await targetQueue.getJobs(states);
-
   const matches = (job: { id?: string | undefined; name?: string }) => {
     if (!search) return true;
     return (
@@ -304,10 +336,31 @@ router.get("/queues", async (req, res) => {
     );
   };
 
+  // Fetch all states in parallel, tag each job with its state
+  const jobsByState = await Promise.all(
+    QUEUE_STATES.map(async (state) => {
+      const jobs = await targetQueue.getJobs([state]);
+      return jobs.map((j) => {
+        const json: Record<string, unknown> = { ...j.asJSON(), _state: state };
+        if (state === "delayed" && j.delay > 0) {
+          json.delayUntil = j.timestamp + j.delay;
+        }
+        return json;
+      });
+    })
+  );
+  const allJobs = jobsByState.flat().filter(matches);
+
+  // Sort: active first, then waiting, delayed, failed, completed
+  const stateOrder: Record<string, number> = {
+    active: 0, waiting: 1, delayed: 2, failed: 3, completed: 4,
+  };
+  allJobs.sort((a, b) => (stateOrder[a._state as string] ?? 9) - (stateOrder[b._state as string] ?? 9));
+
   res.json({
     queues: statsResults,
     selectedQueue: target?.key || "download",
-    jobs: jobs.filter(matches),
+    jobs: allJobs,
   });
 });
 

src/server/routes/repository-private.ts

@@ -21,7 +21,7 @@ import RepositoryModel from "../../core/model/repositories/repositories.model";
 import User from "../../core/User";
 import { RepositoryStatus } from "../../core/types";
 import { IUserDocument } from "../../core/model/users/users.types";
-import { checkToken, octokit } from "../../core/GitHubUtils";
+import { checkToken, octokit, getRedisGateResetAt } from "../../core/GitHubUtils";
 import { createLogger, serializeError } from "../../core/logger";
 
 const logger = createLogger("route:repo");
@@ -294,6 +294,10 @@ router.get("/:repoId/", async (req: express.Request, res: express.Response) => {
         : fullRepo.owner.id === user.model.id
         ? "owner"
         : "coauthor";
+    const gateResetAt = await getRedisGateResetAt();
+    if (gateResetAt > 0) {
+      json.rateLimitResetAt = gateResetAt;
+    }
     res.json(json);
   } catch (error) {
     handleError(error, res, req);

src/server/routes/repository-public.ts

@@ -339,8 +339,8 @@ router.get(
         fiveMinuteAgo.setMinutes(fiveMinuteAgo.getMinutes() - 5);
         if (repo.status != "ready") {
           if (
+            repo.status != RepositoryStatus.QUEUE &&
             repo.model.statusDate < fiveMinuteAgo
-            // && repo.status != "preparing"
           ) {
             await repo.updateStatus(RepositoryStatus.PREPARING);
             await downloadQueue.add(repo.repoId, { repoId: repo.repoId }, {
@@ -359,6 +359,14 @@ router.get(
               }
             );
           }
+          const rlMatch = (repo.model.statusMessage || "").match(/^rate_limited:(\d+)$/);
+          if (rlMatch) {
+            const resetAt = parseInt(rlMatch[1], 10);
+            throw new AnonymousError("rate_limited", {
+              httpStatus: 425,
+              object: { resetAt },
+            });
+          }
           throw new AnonymousError("repository_not_ready", {
             httpStatus: 425,
             object: repo,

src/server/routes/route-utils.ts

@@ -218,7 +218,17 @@ export function handleError(
   if (res && !res.headersSent) {
     const safeCode =
       error instanceof AnonymousError ? errorCode : "internal_error";
-    res.status(status).json({ error: safeCode });
+    const body: Record<string, unknown> = { error: safeCode };
+    if (
+      error instanceof AnonymousError &&
+      safeCode === "rate_limited" &&
+      error.value &&
+      typeof error.value === "object" &&
+      "resetAt" in (error.value as Record<string, unknown>)
+    ) {
+      body.resetAt = (error.value as Record<string, unknown>).resetAt;
+    }
+    res.status(status).json(body);
   }
   return;
 }