CalvinBackup/anonymous_github
1
0
Fork 0
mirror of https://github.com/tdurieux/anonymous_github.git synced 2026-05-24 18:14:03 +02:00
Code Issues Packages Projects Releases Wiki Activity

multiple fixes

Browse Source
This commit is contained in:
tdurieux
2026-05-05 10:32:31 +03:00
parent 5b72b630c4
commit f8c91ca0af
23 changed files with 1675 additions and 661 deletions
Download Patch File Download Diff File Expand all files Collapse all files
package-lock.json
Generated
+8 -7
View File
@@ -12225,9 +12225,10 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
}, },
"node_modules/msgpackr": { "node_modules/msgpackr": {
"version": "1.10.1", "version": "1.11.12",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.1.tgz", "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.12.tgz",
"integrity": "sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==", "integrity": "sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg==",
"license": "MIT",
"optionalDependencies": { "optionalDependencies": {
"msgpackr-extract": "^3.0.2" "msgpackr-extract": "^3.0.2"
} }
@@ -20737,7 +20738,7 @@
"glob": "^8.0.3", "glob": "^8.0.3",
"ioredis": "^5.2.2", "ioredis": "^5.2.2",
"lodash": "^4.17.21", "lodash": "^4.17.21",
"msgpackr": "^1.6.2", "msgpackr": "^1.11.12",
"semver": "^7.3.7", "semver": "^7.3.7",
"tslib": "^2.0.0", "tslib": "^2.0.0",
"uuid": "^9.0.0" "uuid": "^9.0.0"
@@ -23517,9 +23518,9 @@
"integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="
}, },
"msgpackr": { "msgpackr": {
"version": "1.10.1", "version": "1.11.12",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.10.1.tgz", "resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.12.tgz",
"integrity": "sha512-r5VRLv9qouXuLiIBrLpl2d5ZvPt8svdQTl5/vMvE4nzDMyEX4sgW5yWhuBBj5UmgwOTWj8CIdSXn5sAfsHAWIQ==", "integrity": "sha512-RBdJ1Un7yGlXWajrkxcSa93nvQ0w4zBf60c0yYv7YtBelP8H2FA7XsfBbMHtXKXUMUxH7zV3Zuozh+kUQWhHvg==",
"requires": { "requires": {
"msgpackr-extract": "^3.0.2" "msgpackr-extract": "^3.0.2"
} }
package.json
+3
View File
@@ -65,6 +65,9 @@
"ts-custom-error": "^3.3.1", "ts-custom-error": "^3.3.1",
"unzip-stream": "^0.3.1" "unzip-stream": "^0.3.1"
}, },
"overrides": {
"msgpackr": "^1.11.12"
},
"devDependencies": { "devDependencies": {
"@eslint/js": "^10.0.1", "@eslint/js": "^10.0.1",
"@types/archiver": "^5.3.4", "@types/archiver": "^5.3.4",
public/css/all.min.css
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
public/css/style.css
+230 -13
View File
@@ -2438,24 +2438,24 @@ code {
} }
} }
/* Admin stat cards */ /* Admin stat cards — compact, denser packing for many KPIs */
.admin-stats { .admin-stats {
display: grid; display: grid;
grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); grid-template-columns: repeat(auto-fit, minmax(120px, 1fr));
gap: 12px; gap: 8px;
margin-bottom: 20px; margin-bottom: 16px;
} }
.admin-stat-card { .admin-stat-card {
background: var(--paper-card); background: var(--paper-card);
border: 1px solid var(--border-color); border: 1px solid var(--border-color);
border-radius: 10px; border-radius: 8px;
padding: 16px 18px; padding: 10px 12px;
} }
.admin-stat-card .stat-value { .admin-stat-card .stat-value {
font-family: var(--font-serif); font-family: var(--font-serif);
font-size: 1.9rem; font-size: 1.4rem;
font-weight: 400; font-weight: 400;
line-height: 1.1; line-height: 1.1;
color: var(--color); color: var(--color);
@@ -2463,11 +2463,11 @@ code {
.admin-stat-card .stat-label { .admin-stat-card .stat-label {
font-family: var(--font-mono); font-family: var(--font-mono);
font-size: 10.5px; font-size: 9.5px;
color: var(--ink-muted); color: var(--ink-muted);
text-transform: uppercase; text-transform: uppercase;
letter-spacing: 0.08em; letter-spacing: 0.08em;
margin-top: 6px; margin-top: 4px;
} }
/* Admin toolbar — kept as alias to dashboard filter row */ /* Admin toolbar — kept as alias to dashboard filter row */
@@ -2679,8 +2679,9 @@ code {
/* Admin section headers */ /* Admin section headers */
.admin-section-header { .admin-section-header {
display: flex; display: flex;
align-items: baseline; align-items: center;
justify-content: space-between; flex-wrap: wrap;
gap: 8px;
margin: 24px 0 12px; margin: 24px 0 12px;
padding-bottom: 8px; padding-bottom: 8px;
border-bottom: 1px solid var(--border-color); border-bottom: 1px solid var(--border-color);
@@ -3966,10 +3967,16 @@ textarea::selection {
} }
.paper-table.paper-table-repos .paper-table-head, .paper-table.paper-table-repos .paper-table-head,
.paper-table.paper-table-repos .paper-table-row { .paper-table.paper-table-repos .paper-table-row {
grid-template-columns: minmax(280px, 2.4fr) 140px 90px 140px 52px; grid-template-columns: minmax(280px, 2.4fr) minmax(140px, 1fr) 90px 140px 52px;
}
.paper-table.paper-table-repos.has-bulk .paper-table-head,
.paper-table.paper-table-repos.has-bulk .paper-table-row {
grid-template-columns: 28px minmax(280px, 2.4fr) minmax(140px, 1fr) 90px 140px 52px;
gap: 12px;
} }
.paper-table .admin-users-row { .paper-table .admin-users-row {
grid-template-columns: minmax(280px, 2.4fr) 140px 140px 52px !important; grid-template-columns: 28px minmax(280px, 2.4fr) 90px 120px 100px 52px !important;
gap: 12px !important;
} }
.paper-table .paper-table-head { .paper-table .paper-table-head {
background: transparent; background: transparent;
@@ -3982,6 +3989,16 @@ textarea::selection {
padding-top: 8px; padding-top: 8px;
padding-bottom: 8px; padding-bottom: 8px;
} }
.paper-table .paper-table-head .sortable {
cursor: pointer;
user-select: none;
display: inline-flex;
align-items: center;
gap: 4px;
}
.paper-table .paper-table-head .sortable:hover { color: var(--color); }
.paper-table .paper-table-head .sortable.active { color: var(--color); }
.paper-table .paper-table-head .sortable i { font-size: 10px; }
.paper-table .paper-table-head .num, .paper-table .paper-table-head .num,
.paper-table .paper-table-row .num { text-align: left; } .paper-table .paper-table-row .num { text-align: left; }
.paper-table .paper-table-row { .paper-table .paper-table-row {
@@ -4046,6 +4063,206 @@ textarea::selection {
.status-dot.status-preparing { background: #C48A2E; } .status-dot.status-preparing { background: #C48A2E; }
.status-dot.status-removed { background: #9A8F7B; } .status-dot.status-removed { background: #9A8F7B; }
.bulk-bar {
display: flex;
align-items: center;
gap: 8px;
background: var(--paper-card);
border: 1px solid var(--border-color);
border-radius: 8px;
padding: 8px 12px;
margin: 8px 0;
}
/* Tighter admin page header — admin is power-user UI, no need for huge title + lede */
.paper-page.admin-page .paper-page-title { font-size: 1.6rem; margin: 4px 0; }
.paper-page.admin-page .paper-page-lede { display: none; }
.paper-page.admin-page .paper-crumbs { margin-bottom: 4px; }
.paper-page.admin-page .admin-nav { margin: 6px 0 12px; }
/* Admin filter toolbar — single compact line, inline labels (no boxed groups) */
.admin-filter-toolbar {
display: flex;
flex-direction: column;
gap: 6px;
margin: 8px 0 12px;
position: sticky;
top: 0;
z-index: 4;
background: var(--paper-bg, #faf8f3);
padding: 6px 0;
border-bottom: 1px solid var(--border-soft, var(--border-color));
box-shadow: 0 1px 0 var(--paper-bg, #faf8f3); /* paint over content peeking under bottom border on Safari */
}
@media (max-width: 900px) {
.admin-filter-toolbar { position: static; box-shadow: none; }
}
.admin-filter-row {
display: flex;
flex-wrap: wrap;
gap: 6px;
align-items: center;
}
.admin-filter-row .search-wrap { flex: 1; min-width: 220px; }
.admin-filter-row .search-wrap .form-control { height: 34px; }
.admin-filter-inline {
display: inline-flex;
align-items: center;
gap: 4px;
font-size: 12px;
color: var(--ink-muted);
}
.admin-filter-inline > label {
font-family: var(--font-mono);
font-size: 10px;
text-transform: uppercase;
letter-spacing: 0.06em;
color: var(--ink-muted);
margin: 0;
white-space: nowrap;
}
.admin-filter-inline .form-control-sm {
height: 28px;
padding: 2px 8px;
font-size: 12px;
}
.admin-filter-inline input[type="text"].form-control-sm,
.admin-filter-inline input[type="search"].form-control-sm { width: 130px; }
.admin-filter-inline input[type="date"].form-control-sm { width: 130px; }
.admin-filter-inline select.form-control-sm { width: auto; min-width: 88px; }
.admin-filter-inline .arrow { color: var(--ink-muted); font-size: 11px; }
.admin-filter-spacer { flex: 1; }
/* Active-filter chips — visually surface what's currently constraining the view */
.admin-active-chips {
display: flex;
flex-wrap: wrap;
gap: 6px;
align-items: center;
font-size: 12px;
color: var(--ink-muted);
}
.admin-active-chip {
display: inline-flex;
align-items: center;
gap: 4px;
padding: 2px 4px 2px 10px;
border: 1px solid var(--border-color);
border-radius: 999px;
background: var(--hover-bg-color);
font-size: 12px;
color: var(--color);
}
.admin-active-chip .key { color: var(--ink-muted); font-family: var(--font-mono); font-size: 10px; text-transform: uppercase; letter-spacing: 0.06em; }
.admin-active-chip button {
background: none;
border: none;
cursor: pointer;
padding: 0 4px;
color: var(--ink-muted);
font-size: 12px;
border-radius: 50%;
}
.admin-active-chip button:hover { color: #B42318; }
.admin-search-hint {
position: absolute;
right: 10px;
top: 50%;
transform: translateY(-50%);
font-family: var(--font-mono);
font-size: 11px;
color: var(--ink-muted);
pointer-events: none;
border: 1px solid var(--border-color);
border-radius: 4px;
padding: 1px 5px;
background: var(--paper-card);
}
.admin-filter-row .search-wrap { position: relative; }
.admin-filter-row .search-wrap .form-control { padding-right: 40px; }
.paper-table-row.row-selected {
background: rgba(47, 122, 68, 0.06);
}
.queue-state-pills { display: inline-flex; gap: 6px; margin-left: 8px; flex-wrap: wrap; }
.queue-state-pills .pill {
font-size: 11px;
padding: 2px 8px;
border-radius: 999px;
border: 1px solid var(--border-color);
background: var(--paper-card);
color: var(--ink-muted);
}
.queue-state-pills .pill-active { color: #C48A2E; border-color: rgba(196,138,46,0.4); }
.queue-state-pills .pill-failed { color: #B42318; border-color: rgba(180,35,24,0.4); }
.queue-state-pills .pill-completed { color: #2F7A44; border-color: rgba(47,122,68,0.3); }
.queue-state-pills .pill-waiting { color: var(--ink-soft); }
.queue-state-pills .pill-delayed { color: var(--ink-muted); }
/* Compact summary bar — replaces stat cards. Each pill is also a filter shortcut. */
.admin-summary {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 6px;
margin: 8px 0 12px;
font-size: 13px;
}
.admin-summary .summary-total {
font-family: var(--font-serif);
font-size: 1.05rem;
color: var(--color);
margin-right: 8px;
}
.admin-summary .summary-meta {
color: var(--ink-muted);
font-size: 12px;
margin-right: 8px;
}
.admin-summary .summary-pill {
display: inline-flex;
align-items: center;
gap: 6px;
padding: 3px 10px;
border-radius: 999px;
border: 1px solid var(--border-color);
background: var(--paper-card);
color: var(--ink-muted);
cursor: pointer;
font-size: 12px;
user-select: none;
transition: background-color 0.1s, border-color 0.1s;
}
.admin-summary .summary-pill:hover { background: var(--hover-bg-color); }
.admin-summary .summary-pill .count {
font-family: var(--font-mono);
font-variant-numeric: tabular-nums;
color: var(--color);
}
.admin-summary .summary-pill.active {
background: var(--color);
color: var(--paper-bg, #fff);
border-color: var(--color);
}
.admin-summary .summary-pill.active .count { color: inherit; }
.admin-summary .summary-pill.error .count { color: #B42318; }
.admin-summary .summary-pill.warn .count { color: #C48A2E; }
.admin-summary .summary-pill.ok .count { color: #2F7A44; }
.job-progress {
flex: 1;
height: 6px;
background: var(--border-soft, var(--border-color));
border-radius: 999px;
overflow: hidden;
margin-left: 12px;
max-width: 200px;
}
.job-progress-bar { height: 100%; background: #2F7A44; transition: width 0.3s; }
.paper-table-empty { .paper-table-empty {
padding: 48px 4px; padding: 48px 4px;
text-align: center; text-align: center;
public/partials/admin/conferences.htm
+52 -88
View File
@@ -1,7 +1,6 @@
<div class="container paper-page"> <div class="container paper-page admin-page">
<div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Conferences</span></div> <div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Conferences</span></div>
<h1 class="paper-page-title">All <em>conferences</em></h1> <h1 class="paper-page-title">Conferences</h1>
<p class="paper-page-lede">Every venue configured on the platform.</p>
<nav class="admin-nav"> <nav class="admin-nav">
<a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a> <a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a>
@@ -10,91 +9,51 @@
<a href="/admin/queues"><i class="fas fa-tasks"></i> Queues</a> <a href="/admin/queues"><i class="fas fa-tasks"></i> Queues</a>
</nav> </nav>
<div class="admin-stats"> <div class="admin-summary">
<div class="admin-stat-card"> <span class="summary-total">{{total >= 0 ? (total | number) : '…'}}</span>
<div class="stat-value" ng-bind="total >= 0 ? (total | number) : '...'"></div> <span class="summary-pill ok" ng-class="{active: query.status == 'ready'}" ng-click="query.status = query.status == 'ready' ? '' : 'ready'; query.page = 1">Ready <span class="count">{{statusCountFor('ready') | number}}</span></span>
<div class="stat-label">Total conferences</div> <span class="summary-pill warn" ng-class="{active: query.status == 'preparing'}" ng-click="query.status = query.status == 'preparing' ? '' : 'preparing'; query.page = 1">Preparing <span class="count">{{statusCountFor('preparing') | number}}</span></span>
</div> <span class="summary-pill error" ng-class="{active: query.status == 'error'}" ng-click="query.status = query.status == 'error' ? '' : 'error'; query.page = 1">Errored <span class="count">{{statusCountFor('error') | number}}</span></span>
<div class="admin-stat-card"> <span class="summary-pill" ng-class="{active: query.status == 'expired'}" ng-click="query.status = query.status == 'expired' ? '' : 'expired'; query.page = 1">Expired <span class="count">{{statusCountFor('expired') | number}}</span></span>
<div class="stat-value">{{query.page}}/{{totalPage || '...'}}</div> <span class="summary-pill" ng-class="{active: query.status == 'removed'}" ng-click="query.status = query.status == 'removed' ? '' : 'removed'; query.page = 1">Removed <span class="count">{{statusCountFor('removed') | number}}</span></span>
<div class="stat-label">Current page</div>
</div>
</div> </div>
<form class="w-100 dashboard-filter-row" aria-label="Conferences" accept-charset="UTF-8"> <div class="alert alert-danger" ng-if="fetchError" style="margin: 8px 0;">
<div class="search-wrap"> <i class="fas fa-exclamation-triangle"></i> {{fetchError}}
<input </div>
type="search"
class="form-control" <form class="w-100 admin-filter-toolbar" aria-label="Conferences" accept-charset="UTF-8">
aria-label="Search conferences" <div class="admin-filter-row">
placeholder="Search conferences…" <div class="search-wrap">
autocomplete="off" <input type="search" class="form-control" placeholder="Search conferences…" autocomplete="off" ng-model="query.search" />
ng-model="query.search" <span class="admin-search-hint" ng-if="!query.search">/</span>
/> </div>
<span class="admin-filter-spacer"></span>
<button class="btn btn-sm" type="button" ng-click="exportCsv()"><i class="fas fa-file-csv"></i> Export</button>
<span class="admin-filter-inline" aria-label="Pagination">
<button class="btn btn-sm" type="button" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"><i class="fas fa-chevron-left"></i></button>
<span style="font-family: var(--font-mono); font-size: 12px; color: var(--ink-muted);">{{query.page}}/{{totalPage || 1}}</span>
<button class="btn btn-sm" type="button" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage"><i class="fas fa-chevron-right"></i></button>
</span>
</div> </div>
<div class="d-flex flex-wrap" style="gap: 8px; align-items: center;">
<div class="pagination-compact">
<button class="btn btn-sm" type="button" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1">
<i class="fas fa-chevron-left"></i>
</button>
<input type="number" class="form-control form-control-sm" ng-model="query.page" min="1" max="{{totalPage}}" />
<span>/{{totalPage}}</span>
<button class="btn btn-sm" type="button" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage">
<i class="fas fa-chevron-right"></i>
</button>
</div>
<div class="dropdown"> <div class="admin-filter-row" ng-if="chips.length">
<button class="btn dropdown-toggle" type="button" id="dropdownSort" data-toggle="dropdown">Sort</button> <div class="admin-active-chips">
<div class="dropdown-menu" aria-labelledby="dropdownSort"> <span class="admin-active-chip" ng-repeat="chip in chips track by chip.key">
<h6 class="dropdown-header">Sort by</h6> <span class="key">{{chip.label}}</span>
<a class="dropdown-item" href="#" ng-click="query.sort = 'source.conferenceName'"> <span>{{chip.value}}</span>
<i class="fas fa-check" ng-show="query.sort == 'source.conferenceName'"></i> Conference <button type="button" ng-click="clearFilter(chip.key)"><i class="fas fa-times"></i></button>
</a> </span>
<a class="dropdown-item" href="#" ng-click="query.sort = 'anonymizeDate'">
<i class="fas fa-check" ng-show="query.sort == 'anonymizeDate'"></i> Anonymize date
</a>
<a class="dropdown-item" href="#" ng-click="query.sort = 'status'">
<i class="fas fa-check" ng-show="query.sort == 'status'"></i> Status
</a>
</div>
</div>
<div class="dropdown">
<button class="btn dropdown-toggle" type="button" id="dropdownStatus" data-toggle="dropdown">Status</button>
<div class="dropdown-menu" aria-labelledby="dropdownStatus">
<h6 class="dropdown-header">Filter by status</h6>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminConfStatusReady" ng-model="query.ready" />
<label class="form-check-label" for="adminConfStatusReady">Ready</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminConfStatusPreparing" ng-model="query.preparing" />
<label class="form-check-label" for="adminConfStatusPreparing">Preparing</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminConfStatusExpired" ng-model="query.expired" />
<label class="form-check-label" for="adminConfStatusExpired">Expired</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminConfStatusRemoved" ng-model="query.removed" />
<label class="form-check-label" for="adminConfStatusRemoved">Removed</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminConfStatusError" ng-model="query.error" />
<label class="form-check-label" for="adminConfStatusError">Error</label>
</div>
</div>
</div> </div>
</div> </div>
</form> </form>
<div class="paper-table paper-table-conferences w-100" role="table" aria-label="Conferences"> <div class="paper-table paper-table-conferences w-100" role="table" aria-label="Conferences">
<div class="paper-table-head" role="row"> <div class="paper-table-head" role="row">
<div role="columnheader">Conference</div> <div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'name'}" ng-click="sortBy('name')">Conference <i class="fas" ng-class="sortIcon('name')"></i></span></div>
<div role="columnheader">Status</div> <div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'status'}" ng-click="sortBy('status')">Status <i class="fas" ng-class="sortIcon('status')"></i></span></div>
<div role="columnheader" class="num">Repos</div> <div role="columnheader" class="num">Repos</div>
<div role="columnheader">Window</div> <div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'startDate'}" ng-click="sortBy('startDate')">Window <i class="fas" ng-class="sortIcon('startDate')"></i></span></div>
<div role="columnheader" aria-label="Actions"></div> <div role="columnheader" aria-label="Actions"></div>
</div> </div>
<div <div
@@ -112,10 +71,12 @@
</div> </div>
</div> </div>
<div class="cell-status" role="cell"> <div class="cell-status" role="cell">
<span class="status-dot" ng-class="{'status-removed': conference.status == 'removed' || conference.status == 'expired', 'status-ready': conference.status == 'ready'}"></span> <span class="status-dot" ng-class="{'status-removed': conference.status == 'removed' || conference.status == 'expired', 'status-ready': conference.status == 'ready', 'status-error': conference.status == 'error', 'status-preparing': conference.status == 'preparing'}"></span>
<span ng-bind="conference.status | title"></span> <span ng-bind="conference.status | title"></span>
</div> </div>
<div class="cell-views num" role="cell" ng-bind="::conference.repositories.length || 0 | number"></div> <div class="cell-views num" role="cell">
<a ng-href="/admin/?conference={{conference.conferenceID}}" ng-bind="::conference.repositories.length || 0 | number" title="Show repositories in this conference"></a>
</div>
<div class="cell-expires" role="cell">{{conference.startDate | date}} &ndash; {{conference.endDate | date}}</div> <div class="cell-expires" role="cell">{{conference.startDate | date}} &ndash; {{conference.endDate | date}}</div>
<div class="cell-actions" role="cell"> <div class="cell-actions" role="cell">
<div class="dropdown"> <div class="dropdown">
@@ -125,6 +86,7 @@
<div class="dropdown-menu dropdown-menu-right"> <div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item" href="/conference/{{conference.conferenceID}}/edit"><i class="far fa-edit"></i> Edit</a> <a class="dropdown-item" href="/conference/{{conference.conferenceID}}/edit"><i class="far fa-edit"></i> Edit</a>
<a class="dropdown-item" href="/conference/{{conference.conferenceID}}/"><i class="fa fa-eye"></i> View</a> <a class="dropdown-item" href="/conference/{{conference.conferenceID}}/"><i class="fa fa-eye"></i> View</a>
<a class="dropdown-item" href="/admin/?conference={{conference.conferenceID}}"><i class="fas fa-code-branch"></i> View repositories</a>
<div class="dropdown-divider"></div> <div class="dropdown-divider"></div>
<a class="dropdown-item text-danger" href="#" ng-show="conference.status != 'removed'" ng-click="removeConference(conference)"><i class="fas fa-trash-alt"></i> Remove</a> <a class="dropdown-item text-danger" href="#" ng-show="conference.status != 'removed'" ng-click="removeConference(conference)"><i class="fas fa-trash-alt"></i> Remove</a>
</div> </div>
@@ -137,15 +99,17 @@
</div> </div>
</div> </div>
<div class="admin-toolbar" ng-if="totalPage > 1" style="justify-content: center; border-bottom: none;"> <div class="admin-toolbar" style="justify-content: space-between; border-bottom: none;">
<div class="pagination-compact"> <span style="font-size: 12px; color: var(--ink-muted);">{{total | number}} results</span>
<button class="btn btn-sm" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"> <div class="pagination-compact" ng-if="totalPage > 1">
<i class="fas fa-chevron-left"></i> Previous <button class="btn btn-sm" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"><i class="fas fa-chevron-left"></i> Previous</button>
</button> <input type="number" class="form-control form-control-sm" ng-model="query.page" min="1" max="{{totalPage}}" style="width: 56px;" />
<span>Page {{query.page}} of {{totalPage}}</span> <span>of {{totalPage}}</span>
<button class="btn btn-sm" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage"> <button class="btn btn-sm" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage">Next <i class="fas fa-chevron-right"></i></button>
Next <i class="fas fa-chevron-right"></i>
</button>
</div> </div>
<span class="admin-filter-inline">
<label>Per page</label>
<select class="form-control form-control-sm" ng-model="query.limit"><option value="10">10</option><option value="25">25</option><option value="50">50</option><option value="100">100</option></select>
</span>
</div> </div>
</div> </div>
public/partials/admin/queues.htm
+83 -108
View File
@@ -1,7 +1,6 @@
<div class="container paper-page"> <div class="container paper-page admin-page">
<div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Queues</span></div> <div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Queues</span></div>
<h1 class="paper-page-title">Background <em>queues</em></h1> <h1 class="paper-page-title">Queues</h1>
<p class="paper-page-lede">Watch anonymization jobs as they move through the workers.</p>
<nav class="admin-nav"> <nav class="admin-nav">
<a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a> <a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a>
@@ -10,117 +9,93 @@
<a href="/admin/queues" class="active"><i class="fas fa-tasks"></i> Queues</a> <a href="/admin/queues" class="active"><i class="fas fa-tasks"></i> Queues</a>
</nav> </nav>
<div class="admin-stats"> <div class="admin-summary">
<div class="admin-stat-card"> <span class="summary-pill warn" ng-class="{active: query.state == 'active'}" ng-click="query.state = query.state == 'active' ? '' : 'active'">Active <span class="count">{{(counts.download.active || 0) + (counts.remove.active || 0) + (counts.cache.active || 0)}}</span></span>
<div class="stat-value">{{downloadJobs.length || 0}}</div> <span class="summary-pill" ng-class="{active: query.state == 'waiting'}" ng-click="query.state = query.state == 'waiting' ? '' : 'waiting'">Waiting <span class="count">{{(counts.download.waiting || 0) + (counts.remove.waiting || 0) + (counts.cache.waiting || 0)}}</span></span>
<div class="stat-label">Download jobs</div> <span class="summary-pill error" ng-class="{active: query.state == 'failed'}" ng-click="query.state = query.state == 'failed' ? '' : 'failed'">Failed <span class="count">{{(counts.download.failed || 0) + (counts.remove.failed || 0) + (counts.cache.failed || 0)}}</span></span>
</div> <span class="summary-pill ok" ng-class="{active: query.state == 'completed'}" ng-click="query.state = query.state == 'completed' ? '' : 'completed'">Completed <span class="count">{{(counts.download.completed || 0) + (counts.remove.completed || 0) + (counts.cache.completed || 0)}}</span></span>
<div class="admin-stat-card"> <span class="summary-pill" ng-class="{active: query.state == 'delayed'}" ng-click="query.state = query.state == 'delayed' ? '' : 'delayed'">Delayed <span class="count">{{(counts.download.delayed || 0) + (counts.remove.delayed || 0) + (counts.cache.delayed || 0)}}</span></span>
<div class="stat-value">{{removeJobs.length || 0}}</div>
<div class="stat-label">Remove jobs</div>
</div>
<div class="admin-stat-card">
<div class="stat-value">{{removeCaches.length || 0}}</div>
<div class="stat-label">Cache jobs</div>
</div>
</div> </div>
<div class="admin-section-header"> <form class="w-100 admin-filter-toolbar" aria-label="Queue filters">
<h2><i class="fas fa-download"></i> Download jobs</h2> <div class="admin-filter-row">
<span class="section-count">{{downloadJobs.length || 0}}</span> <div class="search-wrap">
</div> <input type="search" class="form-control" placeholder="Search by job/repo id…" ng-model="query.search" autocomplete="off" />
<span class="admin-search-hint" ng-if="!query.search">/</span>
</div>
<span class="admin-filter-inline">
<label>State</label>
<select class="form-control form-control-sm" ng-model="query.state">
<option value="">Any</option>
<option value="waiting">Waiting</option>
<option value="active">Active</option>
<option value="completed">Completed</option>
<option value="failed">Failed</option>
<option value="delayed">Delayed</option>
</select>
</span>
<span class="admin-filter-spacer"></span>
<label class="admin-filter-inline" style="cursor:pointer;">
<input type="checkbox" ng-model="query.autoRefresh" />
Auto-refresh
</label>
<button class="btn btn-sm" type="button" ng-click="refreshNow()" title="Refresh now"><i class="fas fa-sync"></i></button>
</div>
</form>
<div class="queue-job-card" ng-repeat="job in downloadJobs as filteredDownloadJobs"> <div ng-repeat="qInfo in [
<div class="job-header"> {key: 'download', label: 'Download jobs', icon: 'fa-download', jobs: downloadJobs, counts: counts.download},
<div class="job-id"> {key: 'remove', label: 'Remove jobs', icon: 'fa-trash', jobs: removeJobs, counts: counts.remove},
<span class="status-dot" ng-class="{'status-ready': job.progress.status == 'ready', 'status-error': job.progress.status == 'error', 'status-preparing': job.progress.status == 'preparing', 'status-removed': job.progress.status == 'removed'}"></span> {key: 'cache', label: 'Cache cleanup jobs', icon: 'fa-broom', jobs: removeCaches, counts: counts.cache}
<a target="_blank" ng-href="/r/{{job.id}}" ng-bind="job.id"></a> ]">
<span ng-bind="job.progress.status | title" style="font-family: var(--font-sans); color: var(--ink-muted); font-size: 12px;"></span> <div class="admin-section-header">
<h2><i class="fas {{qInfo.icon}}"></i> {{qInfo.label}}</h2>
<span class="section-count">{{qInfo.jobs.length || 0}}</span>
<span class="queue-state-pills">
<span class="pill pill-waiting" ng-if="qInfo.counts.waiting">{{qInfo.counts.waiting}} waiting</span>
<span class="pill pill-active" ng-if="qInfo.counts.active">{{qInfo.counts.active}} active</span>
<span class="pill pill-completed" ng-if="qInfo.counts.completed">{{qInfo.counts.completed}} done</span>
<span class="pill pill-failed" ng-if="qInfo.counts.failed">{{qInfo.counts.failed}} failed</span>
<span class="pill pill-delayed" ng-if="qInfo.counts.delayed">{{qInfo.counts.delayed}} delayed</span>
</span>
<span style="margin-left: auto; display: inline-flex; gap: 6px;">
<button class="btn btn-sm" type="button" ng-click="bulkRetryFailed(qInfo.key)" ng-disabled="!qInfo.counts.failed"><i class="fas fa-redo"></i> Retry all failed</button>
<button class="btn btn-sm text-danger" type="button" ng-click="bulkDrain(qInfo.key)"><i class="fas fa-eraser"></i> Drain</button>
</span>
</div>
<div class="queue-job-card" ng-repeat="job in qInfo.jobs | filter:jobMatchesState as filteredJobs">
<div class="job-header">
<div class="job-id">
<span class="status-dot" ng-class="{'status-ready': job.progress.status == 'ready', 'status-error': job.progress.status == 'error' || (job.stacktrace && job.stacktrace.length), 'status-preparing': job.progress.status == 'preparing' || (job.processedOn && !job.finishedOn), 'status-removed': job.progress.status == 'removed'}"></span>
<a target="_blank" ng-href="/r/{{job.id}}" ng-bind="job.id"></a>
<span ng-bind="job.progress.status | title" style="font-family: var(--font-sans); color: var(--ink-muted); font-size: 12px;"></span>
</div>
<div ng-if="jobProgressPct(job) !== null" class="job-progress" title="{{jobProgressPct(job)}}%">
<div class="job-progress-bar" style="width: {{jobProgressPct(job)}}%;"></div>
</div>
</div>
<div class="job-timestamps">
<span ng-if="job.timestamp"><i class="fas fa-clock"></i> Created: {{job.timestamp | humanTime}}</span>
<span ng-if="job.processedOn"><i class="fas fa-cog"></i> Processed: {{job.processedOn | humanTime}}</span>
<span ng-if="job.finishedOn"><i class="fas fa-check"></i> Finished: {{job.finishedOn | humanTime}}</span>
<span ng-if="job.attemptsMade"><i class="fas fa-redo"></i> Attempts: {{job.attemptsMade}}</span>
</div>
<div ng-if="job.failedReason" style="color:#B42318; font-size:0.8rem; margin-top:4px;">{{job.failedReason}}</div>
<div ng-if="job.stacktrace.length">
<pre ng-repeat="stack in job.stacktrace track by $index" style="font-size: 0.8rem; max-height: 100px; overflow: auto; margin: 6px 0 0 0"><code ng-bind="stack"></code></pre>
</div>
<div class="job-actions">
<button class="btn btn-sm" ng-click="retryJob(qInfo.key, job)"><i class="fas fa-sync"></i> Retry</button>
<button class="btn btn-sm" ng-click="removeJob(qInfo.key, job)"><i class="fas fa-trash-alt"></i> Remove</button>
<a class="btn btn-sm" href="/anonymize/{{job.id}}"><i class="far fa-edit"></i> Edit</a>
</div> </div>
</div> </div>
<div class="job-timestamps">
<span ng-if="job.timestamp"><i class="fas fa-clock"></i> Created: {{job.timestamp | humanTime}}</span>
<span ng-if="job.processedOn"><i class="fas fa-cog"></i> Processed: {{job.processedOn | humanTime}}</span>
<span ng-if="job.finishedOn"><i class="fas fa-check"></i> Finished: {{job.finishedOn | humanTime}}</span>
</div>
<div ng-if="job.stacktrace.length">
<pre ng-repeat="stack in job.stacktrace track by $index" style="font-size: 0.8rem; max-height: 100px; overflow: auto; margin: 6px 0 0 0"><code ng-bind="stack"></code></pre>
</div>
<div class="job-actions">
<button class="btn btn-sm" ng-click="retryJob('download', job)"><i class="fas fa-sync"></i> Retry</button>
<button class="btn btn-sm" ng-click="removeJob('download', job)"><i class="fas fa-trash-alt"></i> Remove</button>
<a class="btn btn-sm" href="/anonymize/{{job.id}}"><i class="far fa-edit"></i> Edit</a>
</div>
</div>
<div class="paper-table-empty" ng-if="filteredDownloadJobs.length == 0" style="border:1px solid var(--border-color);border-radius:10px;background:var(--paper-card);"> <div class="paper-table-empty" ng-if="filteredJobs.length == 0" style="border:1px solid var(--border-color);border-radius:10px;background:var(--paper-card);">
<i class="fas fa-check-circle"></i> <i class="fas fa-check-circle"></i>
<span>No download jobs in the queue.</span> <span ng-if="!query.search && !query.state">No {{qInfo.label | lowercase}} in the queue.</span>
</div> <span ng-if="query.search || query.state">No jobs match the current filters.</span>
<div class="admin-section-header">
<h2><i class="fas fa-trash"></i> Remove jobs</h2>
<span class="section-count">{{removeJobs.length || 0}}</span>
</div>
<div class="queue-job-card" ng-repeat="job in removeJobs as filteredRemoveJobs">
<div class="job-header">
<div class="job-id">
<span class="status-dot" ng-class="{'status-ready': job.progress.status == 'ready', 'status-error': job.progress.status == 'error', 'status-preparing': job.progress.status == 'preparing', 'status-removed': job.progress.status == 'removed'}"></span>
<a target="_blank" ng-href="/r/{{job.id}}" ng-bind="job.id"></a>
<span ng-bind="job.progress.status | title" style="font-family: var(--font-sans); color: var(--ink-muted); font-size: 12px;"></span>
</div>
</div> </div>
<div class="job-timestamps">
<span ng-if="job.timestamp"><i class="fas fa-clock"></i> Created: {{job.timestamp | humanTime}}</span>
<span ng-if="job.processedOn"><i class="fas fa-cog"></i> Processed: {{job.processedOn | humanTime}}</span>
<span ng-if="job.finishedOn"><i class="fas fa-check"></i> Finished: {{job.finishedOn | humanTime}}</span>
</div>
<div ng-if="job.stacktrace.length">
<pre ng-repeat="stack in job.stacktrace track by $index" style="font-size: 0.8rem; max-height: 100px; overflow: auto; margin: 6px 0 0 0"><code ng-bind="stack"></code></pre>
</div>
<div class="job-actions">
<button class="btn btn-sm" ng-click="retryJob('remove', job)"><i class="fas fa-sync"></i> Retry</button>
<button class="btn btn-sm" ng-click="removeJob('remove', job)"><i class="fas fa-trash-alt"></i> Remove</button>
<a class="btn btn-sm" href="/anonymize/{{job.id}}"><i class="far fa-edit"></i> Edit</a>
</div>
</div>
<div class="paper-table-empty" ng-if="filteredRemoveJobs.length == 0" style="border:1px solid var(--border-color);border-radius:10px;background:var(--paper-card);">
<i class="fas fa-check-circle"></i>
<span>No remove jobs in the queue.</span>
</div>
<div class="admin-section-header">
<h2><i class="fas fa-broom"></i> Cache cleanup jobs</h2>
<span class="section-count">{{removeCaches.length || 0}}</span>
</div>
<div class="queue-job-card" ng-repeat="job in removeCaches as filteredRemoveCache">
<div class="job-header">
<div class="job-id">
<span class="status-dot" ng-class="{'status-ready': job.progress.status == 'ready', 'status-error': job.progress.status == 'error', 'status-preparing': job.progress.status == 'preparing', 'status-removed': job.progress.status == 'removed'}"></span>
<a target="_blank" ng-href="/r/{{job.id}}" ng-bind="job.id"></a>
<span ng-bind="job.progress.status | title" style="font-family: var(--font-sans); color: var(--ink-muted); font-size: 12px;"></span>
</div>
</div>
<div class="job-timestamps">
<span ng-if="job.timestamp"><i class="fas fa-clock"></i> Created: {{job.timestamp | humanTime}}</span>
<span ng-if="job.processedOn"><i class="fas fa-cog"></i> Processed: {{job.processedOn | humanTime}}</span>
<span ng-if="job.finishedOn"><i class="fas fa-check"></i> Finished: {{job.finishedOn | humanTime}}</span>
</div>
<div ng-if="job.stacktrace.length">
<pre ng-repeat="stack in job.stacktrace track by $index" style="font-size: 0.8rem; max-height: 100px; overflow: auto; margin: 6px 0 0 0"><code ng-bind="stack"></code></pre>
</div>
<div class="job-actions">
<button class="btn btn-sm" ng-click="retryJob('cache', job)"><i class="fas fa-sync"></i> Retry</button>
<button class="btn btn-sm" ng-click="removeJob('cache', job)"><i class="fas fa-trash-alt"></i> Remove</button>
<a class="btn btn-sm" href="/anonymize/{{job.id}}"><i class="far fa-edit"></i> Edit</a>
</div>
</div>
<div class="paper-table-empty" ng-if="filteredRemoveCache.length == 0" style="border:1px solid var(--border-color);border-radius:10px;background:var(--paper-card);">
<i class="fas fa-check-circle"></i>
<span>No cache cleanup jobs in the queue.</span>
</div> </div>
</div> </div>
public/partials/admin/repositories.htm
+84 -95
View File
@@ -1,7 +1,6 @@
<div class="container paper-page"> <div class="container paper-page admin-page">
<div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Repositories</span></div> <div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Repositories</span></div>
<h1 class="paper-page-title">Anonymized <em>repositories</em></h1> <h1 class="paper-page-title">Repositories</h1>
<p class="paper-page-lede">Every anonymization across every user and conference.</p>
<nav class="admin-nav"> <nav class="admin-nav">
<a href="/admin/" class="active"><i class="fas fa-code-branch"></i> Repositories</a> <a href="/admin/" class="active"><i class="fas fa-code-branch"></i> Repositories</a>
@@ -10,111 +9,84 @@
<a href="/admin/queues"><i class="fas fa-tasks"></i> Queues</a> <a href="/admin/queues"><i class="fas fa-tasks"></i> Queues</a>
</nav> </nav>
<div class="admin-stats"> <div class="admin-summary">
<div class="admin-stat-card"> <span class="summary-total">{{total >= 0 ? (total | number) : '…'}}</span>
<div class="stat-value" ng-bind="total >= 0 ? (total | number) : '...'"></div> <span class="summary-meta">{{totalSize | humanFileSize}} on disk</span>
<div class="stat-label">Total repos</div> <span class="summary-pill ok" ng-class="{active: query.ready}" ng-click="query.ready = !query.ready; query.page = 1" title="Toggle ready filter">Ready <span class="count">{{statusCountFor('ready') | number}}</span></span>
</div> <span class="summary-pill warn" ng-class="{active: query.preparing}" ng-click="query.preparing = !query.preparing; query.page = 1" title="Toggle preparing filter">Preparing <span class="count">{{statusCountFor('preparing') + statusCountFor('download') | number}}</span></span>
<div class="admin-stat-card"> <span class="summary-pill error" ng-class="{active: query.error}" ng-click="query.error = !query.error; query.page = 1" title="Toggle errored filter">Errored <span class="count">{{statusCountFor('error') | number}}</span></span>
<div class="stat-value">{{query.page}}/{{totalPage || '...'}}</div> <span class="summary-pill" ng-class="{active: query.expired}" ng-click="query.expired = !query.expired; query.page = 1" title="Toggle expired filter">Expired <span class="count">{{statusCountFor('expired') + statusCountFor('expiring') | number}}</span></span>
<div class="stat-label">Current page</div> <span class="summary-pill" ng-class="{active: query.removed}" ng-click="query.removed = !query.removed; query.page = 1" title="Toggle removed filter">Removed <span class="count">{{statusCountFor('removed') + statusCountFor('removing') | number}}</span></span>
</div>
</div> </div>
<form class="w-100 dashboard-filter-row" aria-label="Repositories" accept-charset="UTF-8"> <div class="alert alert-danger" ng-if="fetchError" style="margin: 8px 0;">
<div class="search-wrap"> <i class="fas fa-exclamation-triangle"></i> {{fetchError}}
<input </div>
type="search"
class="form-control" <form class="w-100 admin-filter-toolbar" aria-label="Repositories" accept-charset="UTF-8">
aria-label="Search repositories" <!-- Row 1: search + scoped inputs + headline actions -->
placeholder="Search repositories…" <div class="admin-filter-row">
autocomplete="off" <div class="search-wrap">
ng-model="query.search"
/>
</div>
<div class="d-flex flex-wrap" style="gap: 8px; align-items: center;">
<div class="pagination-compact">
<button class="btn btn-sm" type="button" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1">
<i class="fas fa-chevron-left"></i>
</button>
<input <input
type="number" type="search"
class="form-control form-control-sm" class="form-control"
ng-model="query.page" aria-label="Search repositories"
min="1" placeholder="Search repoId, source repo, error message…"
max="{{totalPage}}" autocomplete="off"
ng-model="query.search"
/> />
<span>/{{totalPage}}</span> <span class="admin-search-hint" ng-if="!query.search">/</span>
<button class="btn btn-sm" type="button" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage">
<i class="fas fa-chevron-right"></i>
</button>
</div> </div>
<span class="admin-filter-inline"><label>Owner</label><input type="text" class="form-control form-control-sm" placeholder="username" ng-model="query.owner" /></span>
<span class="admin-filter-inline"><label>Conference</label><input type="text" class="form-control form-control-sm" placeholder="ID" ng-model="query.conference" /></span>
<span class="admin-filter-spacer"></span>
<button class="btn btn-sm" type="button" ng-click="exportCsv()" title="Export current view to CSV"><i class="fas fa-file-csv"></i> Export</button>
<span class="admin-filter-inline" aria-label="Pagination">
<button class="btn btn-sm" type="button" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"><i class="fas fa-chevron-left"></i></button>
<span style="font-family: var(--font-mono); font-size: 12px; color: var(--ink-muted);">{{query.page}}/{{totalPage || 1}}</span>
<button class="btn btn-sm" type="button" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage"><i class="fas fa-chevron-right"></i></button>
</span>
</div>
<div class="dropdown"> <!-- Row 2: appears only when there are active filter chips -->
<button class="btn dropdown-toggle" type="button" id="dropdownSort" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Sort</button> <div class="admin-filter-row" ng-if="chips.length">
<div class="dropdown-menu" aria-labelledby="dropdownSort"> <div class="admin-active-chips">
<h6 class="dropdown-header">Sort by</h6> <span class="admin-active-chip" ng-repeat="chip in chips track by chip.key">
<a class="dropdown-item" href="#" ng-click="query.sort = 'source.repositoryName'"> <span class="key">{{chip.label}}</span>
<i class="fas fa-check" ng-show="query.sort == 'source.repositoryName'"></i> Repository <span>{{chip.value}}</span>
</a> <button type="button" ng-click="clearFilter(chip.key)" aria-label="Remove filter"><i class="fas fa-times"></i></button>
<a class="dropdown-item" href="#" ng-click="query.sort = 'anonymizeDate'"> </span>
<i class="fas fa-check" ng-show="query.sort == 'anonymizeDate'"></i> Anonymize date
</a>
<a class="dropdown-item" href="#" ng-click="query.sort = 'status'">
<i class="fas fa-check" ng-show="query.sort == 'status'"></i> Status
</a>
<a class="dropdown-item" href="#" ng-click="query.sort = 'lastView'">
<i class="fas fa-check" ng-show="query.sort == 'lastView'"></i> Last view
</a>
<a class="dropdown-item" href="#" ng-click="query.sort = 'pageView'">
<i class="fas fa-check" ng-show="query.sort == 'pageView'"></i> Page views
</a>
</div>
</div>
<div class="dropdown">
<button class="btn dropdown-toggle" type="button" id="dropdownStatus" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Status</button>
<div class="dropdown-menu" aria-labelledby="dropdownStatus">
<h6 class="dropdown-header">Filter by status</h6>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminStatusReady" ng-model="query.ready" />
<label class="form-check-label" for="adminStatusReady">Ready</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminStatusPreparing" ng-model="query.preparing" />
<label class="form-check-label" for="adminStatusPreparing">Preparing</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminStatusExpired" ng-model="query.expired" />
<label class="form-check-label" for="adminStatusExpired">Expired</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminStatusRemoved" ng-model="query.removed" />
<label class="form-check-label" for="adminStatusRemoved">Removed</label>
</div>
<div class="form-check dropdown-item">
<input class="form-check-input" type="checkbox" id="adminStatusError" ng-model="query.error" />
<label class="form-check-label" for="adminStatusError">Error</label>
</div>
</div>
</div> </div>
</div> </div>
</form> </form>
<div class="paper-table paper-table-repos w-100" role="table" aria-label="Repositories"> <div class="bulk-bar" ng-if="selectedCount() > 0">
<span><strong>{{selectedCount()}}</strong> selected</span>
<button class="btn btn-sm" type="button" ng-click="bulkRefresh()"><i class="fas fa-sync"></i> Force refresh</button>
<button class="btn btn-sm text-danger" type="button" ng-click="bulkRemoveCache()"><i class="fas fa-broom"></i> Remove cache</button>
<button class="btn btn-sm" type="button" ng-click="clearSelection()">Clear</button>
</div>
<div class="paper-table paper-table-repos has-bulk w-100" role="table" aria-label="Repositories">
<div class="paper-table-head" role="row"> <div class="paper-table-head" role="row">
<div role="columnheader">Repository</div> <div role="columnheader" style="width: 28px;">
<div role="columnheader">Status</div> <input type="checkbox" ng-click="selectAllOnPage()" ng-checked="allSelected" aria-label="Select all on page" />
<div role="columnheader" class="num">Views</div> </div>
<div role="columnheader">Anonymized</div> <div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'source.repositoryName'}" ng-click="sortBy('source.repositoryName')">Repository <i class="fas" ng-class="sortIcon('source.repositoryName')"></i></span></div>
<div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'status'}" ng-click="sortBy('status')">Status <i class="fas" ng-class="sortIcon('status')"></i></span></div>
<div role="columnheader" class="num"><span class="sortable" ng-class="{active: query.sort == 'pageView'}" ng-click="sortBy('pageView')">Views <i class="fas" ng-class="sortIcon('pageView')"></i></span></div>
<div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'anonymizeDate'}" ng-click="sortBy('anonymizeDate')">Anonymized <i class="fas" ng-class="sortIcon('anonymizeDate')"></i></span></div>
<div role="columnheader" aria-label="Actions"></div> <div role="columnheader" aria-label="Actions"></div>
</div> </div>
<div <div
class="paper-table-row" class="paper-table-row"
role="row" role="row"
ng-class="{'repo-inactive': repo.status == 'expired' || repo.status == 'removed'}" ng-class="{'repo-inactive': repo.status == 'expired' || repo.status == 'removed', 'row-selected': selected[repo.repoId]}"
ng-repeat="repo in repositories | filter:repoFiler | orderBy:orderBy as filteredRepositories" ng-repeat="repo in repositories | filter:repoFiler | orderBy:orderBy as filteredRepositories"
> >
<div role="cell" style="width: 28px;">
<input type="checkbox" ng-model="selected[repo.repoId]" aria-label="Select repository" />
</div>
<div class="cell-anon" role="cell"> <div class="cell-anon" role="cell">
<span class="type-badge type-repo">Repo</span> <span class="type-badge type-repo">Repo</span>
<div class="anon-text"> <div class="anon-text">
@@ -125,8 +97,11 @@
</div> </div>
</div> </div>
<div class="cell-status" role="cell"> <div class="cell-status" role="cell">
<span class="status-dot" ng-class="{'status-removed': repo.status == 'removed' || repo.status == 'expired', 'status-ready': repo.status == 'ready', 'status-error': repo.status == 'error', 'status-preparing': repo.status == 'preparing'}"></span> <span class="status-line">
<span ng-bind="repo.status | title"></span> <span class="status-dot" ng-class="{'status-removed': repo.status == 'removed' || repo.status == 'expired', 'status-ready': repo.status == 'ready', 'status-error': repo.status == 'error', 'status-preparing': repo.status == 'preparing'}"></span>
<span ng-bind="repo.status | title"></span>
</span>
<span class="status-sub" ng-if="repo.statusMessage" title="{{repo.statusMessage}}" ng-bind="repo.statusMessage"></span>
</div> </div>
<div class="cell-views num" role="cell" ng-bind="::repo.pageView || 0 | number"></div> <div class="cell-views num" role="cell" ng-bind="::repo.pageView || 0 | number"></div>
<div class="cell-expires" role="cell" ng-bind="repo.anonymizeDate | humanTime"></div> <div class="cell-expires" role="cell" ng-bind="repo.anonymizeDate | humanTime"></div>
@@ -139,9 +114,11 @@
<a class="dropdown-item" href="/anonymize/{{repo.repoId}}"><i class="far fa-edit"></i> Edit</a> <a class="dropdown-item" href="/anonymize/{{repo.repoId}}"><i class="far fa-edit"></i> Edit</a>
<a class="dropdown-item" href="/r/{{repo.repoId}}/"><i class="fa fa-eye"></i> View repo</a> <a class="dropdown-item" href="/r/{{repo.repoId}}/"><i class="fa fa-eye"></i> View repo</a>
<a class="dropdown-item" href="/w/{{repo.repoId}}/" target="_self" ng-if="repo.options.page && repo.status == 'ready'"><i class="fas fa-globe"></i> View page</a> <a class="dropdown-item" href="/w/{{repo.repoId}}/" target="_self" ng-if="repo.options.page && repo.status == 'ready'"><i class="fas fa-globe"></i> View page</a>
<a class="dropdown-item" href="#" ng-click="fetchGithubInfo(repo)"><i class="fab fa-github"></i> Live GitHub info</a>
<div class="dropdown-divider"></div> <div class="dropdown-divider"></div>
<a class="dropdown-item" href="#" ng-show="repo.status == 'ready' || repo.status == 'error'" ng-click="updateRepository(repo)"><i class="fas fa-sync"></i> Force update</a> <a class="dropdown-item" href="#" ng-show="repo.status == 'ready' || repo.status == 'error'" ng-click="updateRepository(repo)"><i class="fas fa-sync"></i> Force update</a>
<a class="dropdown-item" href="#" ng-show="repo.status == 'removed'" ng-click="updateRepository(repo)"><i class="fas fa-check-circle"></i> Enable</a> <a class="dropdown-item" href="#" ng-show="repo.status == 'removed'" ng-click="updateRepository(repo)"><i class="fas fa-check-circle"></i> Enable</a>
<a class="dropdown-item" href="#" ng-show="repo.statusMessage" ng-click="showStatusMessage(repo)"><i class="fas fa-exclamation-triangle"></i> View status message</a>
<div class="dropdown-divider"></div> <div class="dropdown-divider"></div>
<a class="dropdown-item" href="#" ng-click="removeCache(repo)"><i class="fas fa-broom"></i> Remove cache</a> <a class="dropdown-item" href="#" ng-click="removeCache(repo)"><i class="fas fa-broom"></i> Remove cache</a>
<a class="dropdown-item text-danger" href="#" ng-show="repo.status == 'ready'" ng-click="removeRepository(repo)"><i class="fas fa-trash-alt"></i> Remove</a> <a class="dropdown-item text-danger" href="#" ng-show="repo.status == 'ready'" ng-click="removeRepository(repo)"><i class="fas fa-trash-alt"></i> Remove</a>
@@ -155,15 +132,27 @@
</div> </div>
</div> </div>
<div class="admin-toolbar" ng-if="totalPage > 1" style="justify-content: center; border-bottom: none;"> <div class="admin-toolbar" style="justify-content: space-between; border-bottom: none;">
<div class="pagination-compact"> <span style="font-size: 12px; color: var(--ink-muted);">{{total | number}} results</span>
<div class="pagination-compact" ng-if="totalPage > 1">
<button class="btn btn-sm" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"> <button class="btn btn-sm" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1">
<i class="fas fa-chevron-left"></i> Previous <i class="fas fa-chevron-left"></i> Previous
</button> </button>
<span>Page {{query.page}} of {{totalPage}}</span> <input type="number" class="form-control form-control-sm" ng-model="query.page" min="1" max="{{totalPage}}" style="width: 56px;" aria-label="Page" />
<span>of {{totalPage}}</span>
<button class="btn btn-sm" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage"> <button class="btn btn-sm" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage">
Next <i class="fas fa-chevron-right"></i> Next <i class="fas fa-chevron-right"></i>
</button> </button>
</div> </div>
<span class="admin-filter-inline">
<label>Per page</label>
<select class="form-control form-control-sm" ng-model="query.limit">
<option value="10">10</option>
<option value="25">25</option>
<option value="50">50</option>
<option value="100">100</option>
<option value="250">250</option>
</select>
</span>
</div> </div>
</div> </div>
public/partials/admin/user.htm
+2 -3
View File
@@ -1,7 +1,6 @@
<div class="container paper-page"> <div class="container paper-page admin-page">
<div class="paper-crumbs"><a href="/admin/users">Users</a> &nbsp;/&nbsp; <span class="here">{{userInfo.username || 'Profile'}}</span></div> <div class="paper-crumbs"><a href="/admin/users">Users</a> &nbsp;/&nbsp; <span class="here">{{userInfo.username || 'Profile'}}</span></div>
<h1 class="paper-page-title">User <em>profile</em></h1> <h1 class="paper-page-title">{{userInfo.username || 'User'}}</h1>
<p class="paper-page-lede">Inspect activity, quota, and repositories for a single account.</p>
<nav class="admin-nav"> <nav class="admin-nav">
<a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a> <a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a>
public/partials/admin/users.htm
+68 -55
View File
@@ -1,7 +1,6 @@
<div class="container paper-page"> <div class="container paper-page admin-page">
<div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Users</span></div> <div class="paper-crumbs">Admin &nbsp;/&nbsp; <span class="here">Users</span></div>
<h1 class="paper-page-title">Registered <em>users</em></h1> <h1 class="paper-page-title">Users</h1>
<p class="paper-page-lede">Browse, search, and manage every account.</p>
<nav class="admin-nav"> <nav class="admin-nav">
<a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a> <a href="/admin/"><i class="fas fa-code-branch"></i> Repositories</a>
@@ -10,73 +9,84 @@
<a href="/admin/queues"><i class="fas fa-tasks"></i> Queues</a> <a href="/admin/queues"><i class="fas fa-tasks"></i> Queues</a>
</nav> </nav>
<div class="admin-stats"> <div class="admin-summary">
<div class="admin-stat-card"> <span class="summary-total">{{total >= 0 ? (total | number) : '…'}}</span>
<div class="stat-value" ng-bind="total >= 0 ? (total | number) : '...'"></div> <span class="summary-pill ok" ng-class="{active: query.status == 'active'}" ng-click="query.status = query.status == 'active' ? '' : 'active'; query.page = 1">Active <span class="count">{{statusCountFor('active') | number}}</span></span>
<div class="stat-label">Total users</div> <span class="summary-pill error" ng-class="{active: query.status == 'banned'}" ng-click="query.status = query.status == 'banned' ? '' : 'banned'; query.page = 1">Banned <span class="count">{{statusCountFor('banned') | number}}</span></span>
</div> <span class="summary-pill" ng-class="{active: query.status == 'removed'}" ng-click="query.status = query.status == 'removed' ? '' : 'removed'; query.page = 1">Removed <span class="count">{{statusCountFor('removed') | number}}</span></span>
<div class="admin-stat-card">
<div class="stat-value">{{query.page}}/{{totalPage || '...'}}</div>
<div class="stat-label">Current page</div>
</div>
</div> </div>
<form class="w-100 dashboard-filter-row" aria-label="Users" accept-charset="UTF-8"> <div class="alert alert-danger" ng-if="fetchError" style="margin: 8px 0;">
<div class="search-wrap"> <i class="fas fa-exclamation-triangle"></i> {{fetchError}}
<input </div>
type="search"
class="form-control"
aria-label="Search users"
placeholder="Search users…"
autocomplete="off"
ng-model="query.search"
/>
</div>
<div class="d-flex flex-wrap" style="gap: 8px; align-items: center;">
<div class="pagination-compact">
<button class="btn btn-sm" type="button" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1">
<i class="fas fa-chevron-left"></i>
</button>
<input type="number" class="form-control form-control-sm" ng-model="query.page" min="1" max="{{totalPage}}" />
<span>/{{totalPage}}</span>
<button class="btn btn-sm" type="button" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage">
<i class="fas fa-chevron-right"></i>
</button>
</div>
<div class="dropdown"> <form class="w-100 admin-filter-toolbar" aria-label="Users" accept-charset="UTF-8">
<button class="btn dropdown-toggle" type="button" id="dropdownSort" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">Sort</button> <div class="admin-filter-row">
<div class="dropdown-menu" aria-labelledby="dropdownSort"> <div class="search-wrap">
<h6 class="dropdown-header">Sort by</h6> <input type="search" class="form-control" aria-label="Search users" placeholder="Search username or email…" autocomplete="off" ng-model="query.search" />
<a class="dropdown-item" href="#" ng-click="query.sort = 'username'"> <span class="admin-search-hint" ng-if="!query.search">/</span>
<i class="fas fa-check" ng-show="query.sort == 'username'"></i> Username </div>
</a> <span class="admin-filter-inline"><label>Role</label>
</div> <select class="form-control form-control-sm" ng-model="query.role"><option value="">Any</option><option value="admin">Admin</option></select>
</span>
<span class="admin-filter-spacer"></span>
<button class="btn btn-sm" type="button" ng-click="exportCsv()"><i class="fas fa-file-csv"></i> Export</button>
<span class="admin-filter-inline" aria-label="Pagination">
<button class="btn btn-sm" type="button" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"><i class="fas fa-chevron-left"></i></button>
<span style="font-family: var(--font-mono); font-size: 12px; color: var(--ink-muted);">{{query.page}}/{{totalPage || 1}}</span>
<button class="btn btn-sm" type="button" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage"><i class="fas fa-chevron-right"></i></button>
</span>
</div>
<div class="admin-filter-row" ng-if="chips.length">
<div class="admin-active-chips">
<span class="admin-active-chip" ng-repeat="chip in chips track by chip.key">
<span class="key">{{chip.label}}</span>
<span>{{chip.value}}</span>
<button type="button" ng-click="clearFilter(chip.key)"><i class="fas fa-times"></i></button>
</span>
</div> </div>
</div> </div>
</form> </form>
<div class="paper-table w-100" role="table" aria-label="Users" style="--cols: minmax(280px, 2.4fr) 140px 140px 52px;"> <div class="bulk-bar" ng-if="selectedCount() > 0">
<span><strong>{{selectedCount()}}</strong> selected</span>
<button class="btn btn-sm text-danger" type="button" ng-click="bulkBan()"><i class="fas fa-ban"></i> Ban</button>
<button class="btn btn-sm" type="button" ng-click="selected = {}; allSelected = false">Clear</button>
</div>
<div class="paper-table w-100" role="table" aria-label="Users" style="--cols: 28px minmax(280px, 2.4fr) 100px 140px 140px 52px;">
<div class="paper-table-head admin-users-row" role="row"> <div class="paper-table-head admin-users-row" role="row">
<div role="columnheader">User</div> <div role="columnheader" style="width: 28px;">
<div role="columnheader">Status</div> <input type="checkbox" ng-click="selectAllOnPage()" ng-checked="allSelected" aria-label="Select all on page" />
</div>
<div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'username'}" ng-click="sortBy('username')">User <i class="fas" ng-class="sortIcon('username')"></i></span></div>
<div role="columnheader" class="num">Repos</div>
<div role="columnheader"><span class="sortable" ng-class="{active: query.sort == 'status'}" ng-click="sortBy('status')">Status <i class="fas" ng-class="sortIcon('status')"></i></span></div>
<div role="columnheader">Role</div> <div role="columnheader">Role</div>
<div role="columnheader" aria-label="Actions"></div> <div role="columnheader" aria-label="Actions"></div>
</div> </div>
<div <div
class="paper-table-row admin-users-row" class="paper-table-row admin-users-row"
role="row" role="row"
ng-class="{'row-selected': selected[u.username]}"
ng-repeat="u in users | filter:userFiler | orderBy:orderBy as filteredUsers" ng-repeat="u in users | filter:userFiler | orderBy:orderBy as filteredUsers"
> >
<div role="cell" style="width: 28px;">
<input type="checkbox" ng-model="selected[u.username]" aria-label="Select user" />
</div>
<div class="cell-anon" role="cell"> <div class="cell-anon" role="cell">
<img ng-src="{{u.photo}}" ng-if="u.photo" width="28" height="28" class="rounded-circle" style="flex-shrink:0;" /> <img ng-src="{{u.photo}}" ng-if="u.photo" width="28" height="28" class="rounded-circle" style="flex-shrink:0;" />
<div class="anon-text"> <div class="anon-text">
<a class="repo-name" ng-href="/admin/users/{{u.username}}" ng-bind="u.username"></a> <a class="repo-name" ng-href="/admin/users/{{u.username}}" ng-bind="u.username"></a>
<div class="anon-sub"> <div class="anon-sub">
<span ng-if="u.emails[0].email">{{u.emails[0].email}}</span><span ng-if="u.emails[0].email">&nbsp;&middot;&nbsp;</span><a href="https://github.com/{{u.username}}" target="_blank"><i class="fab fa-github"></i> {{u.username}}</a> <span ng-if="u.emails[0].email">{{u.emails[0].email}}</span><span ng-if="u.emails[0].email">&nbsp;&middot;&nbsp;</span><a href="https://github.com/{{u.username}}" target="_blank"><i class="fab fa-github"></i> {{u.username}}</a><span ng-if="u.dateOfEntry">&nbsp;&middot;&nbsp;Joined {{u.dateOfEntry | humanTime}}</span>
</div> </div>
</div> </div>
</div> </div>
<div class="cell-views num" role="cell">
<a ng-href="/admin/?owner={{u.username}}" ng-bind="(u.repoCount || 0) | number" title="Show this user's repositories"></a>
</div>
<div class="cell-status" role="cell"> <div class="cell-status" role="cell">
<span class="status-dot" ng-class="{'status-ready': u.status == 'active', 'status-removed': u.status == 'removed' || u.status == 'banned'}"></span> <span class="status-dot" ng-class="{'status-ready': u.status == 'active', 'status-removed': u.status == 'removed' || u.status == 'banned'}"></span>
<span ng-bind="u.status | title"></span> <span ng-bind="u.status | title"></span>
@@ -92,6 +102,7 @@
</button> </button>
<div class="dropdown-menu dropdown-menu-right"> <div class="dropdown-menu dropdown-menu-right">
<a class="dropdown-item" href="/admin/users/{{u.username}}"><i class="far fa-eye"></i> View details</a> <a class="dropdown-item" href="/admin/users/{{u.username}}"><i class="far fa-eye"></i> View details</a>
<a class="dropdown-item" href="/admin/?owner={{u.username}}"><i class="fas fa-code-branch"></i> View repositories</a>
<div class="dropdown-divider"></div> <div class="dropdown-divider"></div>
<a class="dropdown-item text-danger" href="#" ng-show="u.status == 'active'" ng-click="banUser(u)"><i class="fas fa-ban"></i> Ban</a> <a class="dropdown-item text-danger" href="#" ng-show="u.status == 'active'" ng-click="banUser(u)"><i class="fas fa-ban"></i> Ban</a>
<a class="dropdown-item" href="#" ng-show="u.status == 'removed' || u.status == 'banned'" ng-click="activateUser(u)"><i class="fas fa-check-circle"></i> Activate</a> <a class="dropdown-item" href="#" ng-show="u.status == 'removed' || u.status == 'banned'" ng-click="activateUser(u)"><i class="fas fa-check-circle"></i> Activate</a>
@@ -105,15 +116,17 @@
</div> </div>
</div> </div>
<div class="admin-toolbar" ng-if="totalPage > 1" style="justify-content: center; border-bottom: none;"> <div class="admin-toolbar" style="justify-content: space-between; border-bottom: none;">
<div class="pagination-compact"> <span style="font-size: 12px; color: var(--ink-muted);">{{total | number}} results</span>
<button class="btn btn-sm" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"> <div class="pagination-compact" ng-if="totalPage > 1">
<i class="fas fa-chevron-left"></i> Previous <button class="btn btn-sm" ng-click="query.page = Math.max(1, query.page - 1)" ng-disabled="query.page <= 1"><i class="fas fa-chevron-left"></i> Previous</button>
</button> <input type="number" class="form-control form-control-sm" ng-model="query.page" min="1" max="{{totalPage}}" style="width: 56px;" />
<span>Page {{query.page}} of {{totalPage}}</span> <span>of {{totalPage}}</span>
<button class="btn btn-sm" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage"> <button class="btn btn-sm" ng-click="query.page = Math.min(totalPage, query.page + 1)" ng-disabled="query.page >= totalPage">Next <i class="fas fa-chevron-right"></i></button>
Next <i class="fas fa-chevron-right"></i>
</button>
</div> </div>
<span class="admin-filter-inline">
<label>Per page</label>
<select class="form-control form-control-sm" ng-model="query.limit"><option value="10">10</option><option value="25">25</option><option value="50">50</option><option value="100">100</option></select>
</span>
</div> </div>
</div> </div>
public/script/admin.js
+396 -4
View File
@@ -18,12 +18,114 @@ angular
$scope.repositories = []; $scope.repositories = [];
$scope.total = -1; $scope.total = -1;
$scope.totalPage = 0; $scope.totalPage = 0;
$scope.statusCounts = [];
$scope.totalSize = 0;
$scope.selected = {};
$scope.allSelected = false;
// Slash-to-focus the search input
const searchKeyHandler = (e) => {
if (e.key === "/" && !["INPUT","TEXTAREA","SELECT"].includes(document.activeElement?.tagName)) {
e.preventDefault();
const el = document.querySelector('.admin-filter-toolbar input[type="search"]');
el && el.focus();
}
};
document.addEventListener("keydown", searchKeyHandler);
$scope.$on("$destroy", () => document.removeEventListener("keydown", searchKeyHandler));
$scope.clearFilter = (key) => {
if (key === "dateRange") { $scope.query.dateFrom = ""; $scope.query.dateTo = ""; }
else $scope.query[key] = "";
$scope.query.page = 1;
};
$scope.chips = [];
const recomputeChips = () => {
const out = [];
if ($scope.query.owner) out.push({ key: "owner", label: "Owner", value: $scope.query.owner });
if ($scope.query.conference) out.push({ key: "conference", label: "Conference", value: $scope.query.conference });
$scope.chips = out;
};
$scope.showStatusMessage = (repo) => {
const msg = repo.statusMessage || "(no message)";
window.prompt(`Status message for ${repo.repoId} (${repo.status}):`, msg);
};
$scope.fetchGithubInfo = (repo) => {
const w = window.open("", "_blank");
if (w) w.document.write("<pre>Loading GitHub info for " + repo.repoId + "...</pre>");
$http.get("/api/admin/repos/" + repo.repoId + "/github").then(
(res) => {
if (w) {
w.document.open();
w.document.write(
"<pre style=\"font:13px monospace;padding:16px;white-space:pre-wrap\">" +
JSON.stringify(res.data, null, 2).replace(/[<>]/g, (c) => c === "<" ? "&lt;" : "&gt;") +
"</pre>"
);
w.document.close();
}
},
(err) => {
const msg = err && err.data ? JSON.stringify(err.data, null, 2) : String(err);
if (w) w.document.body.innerHTML = "<pre style=\"color:#B42318;padding:16px\">" + msg + "</pre>";
}
);
};
$scope.statusCountFor = (s) => {
const row = ($scope.statusCounts || []).find((c) => c._id === s);
return row ? row.count : 0;
};
$scope.statusStorageFor = (s) => {
const row = ($scope.statusCounts || []).find((c) => c._id === s);
return row ? row.storage : 0;
};
$scope.isErrorsOnly = () =>
$scope.query &&
$scope.query.error && !$scope.query.ready && !$scope.query.preparing &&
!$scope.query.expired && !$scope.query.removed;
$scope.toggleErrorsOnly = () => {
if ($scope.isErrorsOnly()) {
Object.assign($scope.query, { ready: false, preparing: true, expired: false, removed: false, error: true });
} else {
Object.assign($scope.query, { ready: false, preparing: false, expired: false, removed: false, error: true });
}
$scope.query.page = 1;
};
$scope.toggleSortDirection = () => {
$scope.query.direction = $scope.query.direction === "asc" ? "desc" : "asc";
};
$scope.sortBy = (field) => {
if ($scope.query.sort === field) {
$scope.query.direction = $scope.query.direction === "asc" ? "desc" : "asc";
} else {
$scope.query.sort = field;
$scope.query.direction = "desc";
}
$scope.query.page = 1;
};
$scope.sortIcon = (field) =>
$scope.query.sort === field
? ($scope.query.direction === "asc" ? "fa-arrow-up" : "fa-arrow-down")
: "";
const reposAdminPrefsKey = "admin.repos.filterPrefs"; const reposAdminPrefsKey = "admin.repos.filterPrefs";
const reposAdminDefaults = { const reposAdminDefaults = {
page: 1, page: 1,
limit: 25, limit: 25,
sort: "lastView", sort: "lastView",
direction: "desc",
search: "", search: "",
owner: "",
conference: "",
dateFrom: "",
dateTo: "",
ready: false, ready: false,
expired: false, expired: false,
removed: false, removed: false,
@@ -36,6 +138,70 @@ angular
search: "", search: "",
}); });
// pre-fill owner / conference from URL ?owner= / ?conference=
const params = new URLSearchParams(window.location.search);
if (params.get("owner")) $scope.query.owner = params.get("owner");
if (params.get("conference")) $scope.query.conference = params.get("conference");
// -------- presets --------
const presetsKey = "admin.repos.presets";
$scope.presets = JSON.parse(localStorage.getItem(presetsKey) || "[]");
$scope.savePreset = () => {
const name = window.prompt("Preset name:");
if (!name) return;
const snapshot = Object.assign({}, $scope.query);
delete snapshot.page;
$scope.presets = ($scope.presets || []).filter((p) => p.name !== name);
$scope.presets.push({ name, query: snapshot });
localStorage.setItem(presetsKey, JSON.stringify($scope.presets));
};
$scope.applyPreset = (p) => {
Object.assign($scope.query, p.query, { page: 1 });
};
$scope.deletePreset = (p) => {
$scope.presets = ($scope.presets || []).filter((x) => x.name !== p.name);
localStorage.setItem(presetsKey, JSON.stringify($scope.presets));
};
// -------- selection / bulk --------
$scope.selectAllOnPage = () => {
$scope.allSelected = !$scope.allSelected;
$scope.repositories.forEach((r) => {
$scope.selected[r.repoId] = $scope.allSelected;
});
};
$scope.selectedCount = () =>
Object.values($scope.selected || {}).filter(Boolean).length;
$scope.selectedRepos = () =>
$scope.repositories.filter((r) => $scope.selected[r.repoId]);
$scope.bulkRefresh = () => {
const repos = $scope.selectedRepos();
if (!repos.length) return;
if (!confirm(`Force refresh ${repos.length} repositories?`)) return;
repos.forEach((r) => $scope.updateRepository(r));
};
$scope.bulkRemoveCache = () => {
const repos = $scope.selectedRepos();
if (!repos.length) return;
if (!confirm(`Purge cache for ${repos.length} repositories?`)) return;
repos.forEach((r) => $scope.removeCache(r));
};
$scope.clearSelection = () => {
$scope.selected = {};
$scope.allSelected = false;
};
// -------- export --------
$scope.exportCsv = () => {
const params = new URLSearchParams(
Object.entries($scope.query).filter(([, v]) => v !== "" && v !== false && v != null)
);
params.set("format", "csv");
params.set("limit", "10000");
window.open("/api/admin/repos?" + params.toString(), "_blank");
};
$scope.removeCache = (repo) => { $scope.removeCache = (repo) => {
$http.delete("/api/admin/repos/" + repo.repoId).then( $http.delete("/api/admin/repos/" + repo.repoId).then(
(res) => { (res) => {
@@ -54,7 +220,6 @@ angular
body: `The repository ${repo.repoId} is going to be refreshed.`, body: `The repository ${repo.repoId} is going to be refreshed.`,
}; };
$scope.toasts.push(toast); $scope.toasts.push(toast);
repo.s;
$http.post(`/api/repo/${repo.repoId}/refresh`).then( $http.post(`/api/repo/${repo.repoId}/refresh`).then(
(res) => { (res) => {
@@ -71,14 +236,20 @@ angular
); );
}; };
$scope.fetchError = null;
function getRepositories() { function getRepositories() {
$scope.fetchError = null;
$http.get("/api/admin/repos", { params: $scope.query }).then( $http.get("/api/admin/repos", { params: $scope.query }).then(
(res) => { (res) => {
$scope.total = res.data.total; $scope.total = res.data.total;
$scope.totalPage = Math.ceil(res.data.total / $scope.query.limit); $scope.totalPage = Math.ceil(res.data.total / $scope.query.limit);
$scope.repositories = res.data.results; $scope.repositories = res.data.results;
$scope.statusCounts = res.data.statusCounts || [];
$scope.totalSize = res.data.totalSize || 0;
$scope.allSelected = false;
}, },
(err) => { (err) => {
$scope.fetchError = (err && err.data && err.data.error) || "Failed to load repositories";
console.error(err); console.error(err);
} }
); );
@@ -93,9 +264,11 @@ angular
timeClear = setTimeout(getRepositories, 500); timeClear = setTimeout(getRepositories, 500);
const { page, search, ...persisted } = $scope.query; const { page, search, ...persisted } = $scope.query;
saveFilterPrefs(reposAdminPrefsKey, persisted); saveFilterPrefs(reposAdminPrefsKey, persisted);
recomputeChips();
}, },
true true
); );
recomputeChips();
}, },
]) ])
.controller("usersAdminController", [ .controller("usersAdminController", [
@@ -116,12 +289,65 @@ angular
$scope.users = []; $scope.users = [];
$scope.total = -1; $scope.total = -1;
$scope.totalPage = 0; $scope.totalPage = 0;
$scope.statusCounts = [];
$scope.selected = {};
$scope.allSelected = false;
const searchKeyHandler = (e) => {
if (e.key === "/" && !["INPUT","TEXTAREA","SELECT"].includes(document.activeElement?.tagName)) {
e.preventDefault();
const el = document.querySelector('.admin-filter-toolbar input[type="search"]');
el && el.focus();
}
};
document.addEventListener("keydown", searchKeyHandler);
$scope.$on("$destroy", () => document.removeEventListener("keydown", searchKeyHandler));
$scope.clearFilter = (key) => {
if (key === "dateRange") { $scope.query.dateFrom = ""; $scope.query.dateTo = ""; }
else $scope.query[key] = "";
$scope.query.page = 1;
};
$scope.chips = [];
const recomputeChipsUsers = () => {
const out = [];
if ($scope.query.role) out.push({ key: "role", label: "Role", value: $scope.query.role });
$scope.chips = out;
};
$scope.statusCountFor = (s) => {
const row = ($scope.statusCounts || []).find((c) => c._id === s);
return row ? row.count : 0;
};
$scope.toggleSortDirection = () => {
$scope.query.direction = $scope.query.direction === "asc" ? "desc" : "asc";
};
$scope.sortBy = (field) => {
if ($scope.query.sort === field) {
$scope.query.direction = $scope.query.direction === "asc" ? "desc" : "asc";
} else {
$scope.query.sort = field;
$scope.query.direction = "desc";
}
$scope.query.page = 1;
};
$scope.sortIcon = (field) =>
$scope.query.sort === field
? ($scope.query.direction === "asc" ? "fa-arrow-up" : "fa-arrow-down")
: "";
const usersAdminPrefsKey = "admin.users.filterPrefs"; const usersAdminPrefsKey = "admin.users.filterPrefs";
const usersAdminDefaults = { const usersAdminDefaults = {
page: 1, page: 1,
limit: 25, limit: 25,
sort: "username", sort: "username",
direction: "asc",
search: "", search: "",
status: "",
role: "",
dateFrom: "",
dateTo: "",
}; };
const savedUsersAdminPrefs = loadFilterPrefs(usersAdminPrefsKey) || {}; const savedUsersAdminPrefs = loadFilterPrefs(usersAdminPrefsKey) || {};
$scope.query = Object.assign({}, usersAdminDefaults, savedUsersAdminPrefs, { $scope.query = Object.assign({}, usersAdminDefaults, savedUsersAdminPrefs, {
@@ -129,15 +355,58 @@ angular
search: "", search: "",
}); });
$scope.selectAllOnPage = () => {
$scope.allSelected = !$scope.allSelected;
$scope.users.forEach((u) => {
$scope.selected[u.username] = $scope.allSelected;
});
};
$scope.selectedCount = () =>
Object.values($scope.selected || {}).filter(Boolean).length;
$scope.selectedUsers = () =>
$scope.users.filter((u) => $scope.selected[u.username]);
$scope.banUser = (u) => {
if (!confirm(`Ban user ${u.username}?`)) return;
$http
.post(`/api/admin/users/${u.username}/ban`)
.then(getUsers, (err) => console.error(err));
};
$scope.activateUser = (u) => {
$http
.post(`/api/admin/users/${u.username}/activate`)
.then(getUsers, (err) => console.error(err));
};
$scope.bulkBan = () => {
const users = $scope.selectedUsers();
if (!users.length) return;
if (!confirm(`Ban ${users.length} users?`)) return;
users.forEach((u) => $scope.banUser(u));
};
$scope.exportCsv = () => {
const params = new URLSearchParams(
Object.entries($scope.query).filter(([, v]) => v !== "" && v !== false && v != null)
);
params.set("format", "csv");
params.set("limit", "10000");
window.open("/api/admin/users?" + params.toString(), "_blank");
};
$scope.fetchError = null;
function getUsers() { function getUsers() {
$scope.fetchError = null;
$http.get("/api/admin/users", { params: $scope.query }).then( $http.get("/api/admin/users", { params: $scope.query }).then(
(res) => { (res) => {
$scope.total = res.data.total; $scope.total = res.data.total;
$scope.totalPage = Math.ceil(res.data.total / $scope.query.limit); $scope.totalPage = Math.ceil(res.data.total / $scope.query.limit);
$scope.users = res.data.results; $scope.users = res.data.results;
$scope.statusCounts = res.data.statusCounts || [];
$scope.allSelected = false;
$scope.$apply(); $scope.$apply();
}, },
(err) => { (err) => {
$scope.fetchError = (err && err.data && err.data.error) || "Failed to load users";
console.error(err); console.error(err);
} }
); );
@@ -152,9 +421,11 @@ angular
timeClear = setTimeout(getUsers, 500); timeClear = setTimeout(getUsers, 500);
const { page, search, ...persisted } = $scope.query; const { page, search, ...persisted } = $scope.query;
saveFilterPrefs(usersAdminPrefsKey, persisted); saveFilterPrefs(usersAdminPrefsKey, persisted);
recomputeChipsUsers();
}, },
true true
); );
recomputeChipsUsers();
}, },
]) ])
.controller("userAdminController", [ .controller("userAdminController", [
@@ -178,7 +449,7 @@ angular
const adminUserPrefsKey = "admin.user.filterPrefs"; const adminUserPrefsKey = "admin.user.filterPrefs";
const adminUserDefaults = { const adminUserDefaults = {
filters: { status: { ready: true, expired: false, removed: false } }, filters: { status: { ready: true, expired: true, removed: true, error: true, preparing: true } },
orderBy: "-anonymizeDate", orderBy: "-anonymizeDate",
}; };
const savedAdminUserPrefs = loadFilterPrefs(adminUserPrefsKey) || {}; const savedAdminUserPrefs = loadFilterPrefs(adminUserPrefsKey) || {};
@@ -297,7 +568,6 @@ angular
body: `The repository ${repo.repoId} is going to be refreshed.`, body: `The repository ${repo.repoId} is going to be refreshed.`,
}; };
$scope.toasts.push(toast); $scope.toasts.push(toast);
repo.s;
$http.post(`/api/repo/${repo.repoId}/refresh`).then( $http.post(`/api/repo/${repo.repoId}/refresh`).then(
(res) => { (res) => {
@@ -355,12 +625,60 @@ angular
$scope.conferences = []; $scope.conferences = [];
$scope.total = -1; $scope.total = -1;
$scope.totalPage = 0; $scope.totalPage = 0;
$scope.statusCounts = [];
const searchKeyHandler = (e) => {
if (e.key === "/" && !["INPUT","TEXTAREA","SELECT"].includes(document.activeElement?.tagName)) {
e.preventDefault();
const el = document.querySelector('.admin-filter-toolbar input[type="search"]');
el && el.focus();
}
};
document.addEventListener("keydown", searchKeyHandler);
$scope.$on("$destroy", () => document.removeEventListener("keydown", searchKeyHandler));
$scope.clearFilter = (key) => {
if (key === "dateRange") { $scope.query.dateFrom = ""; $scope.query.dateTo = ""; }
else $scope.query[key] = "";
$scope.query.page = 1;
};
$scope.chips = [];
const recomputeChipsConf = () => {
$scope.chips = [];
};
$scope.statusCountFor = (s) => {
const row = ($scope.statusCounts || []).find((c) => c._id === s);
return row ? row.count : 0;
};
$scope.toggleSortDirection = () => {
$scope.query.direction = $scope.query.direction === "asc" ? "desc" : "asc";
};
$scope.sortBy = (field) => {
if ($scope.query.sort === field) {
$scope.query.direction = $scope.query.direction === "asc" ? "desc" : "asc";
} else {
$scope.query.sort = field;
$scope.query.direction = "desc";
}
$scope.query.page = 1;
};
$scope.sortIcon = (field) =>
$scope.query.sort === field
? ($scope.query.direction === "asc" ? "fa-arrow-up" : "fa-arrow-down")
: "";
const confAdminPrefsKey = "admin.conferences.filterPrefs"; const confAdminPrefsKey = "admin.conferences.filterPrefs";
const confAdminDefaults = { const confAdminDefaults = {
page: 1, page: 1,
limit: 25, limit: 25,
sort: "name", sort: "name",
direction: "asc",
search: "", search: "",
status: "",
dateFrom: "",
dateTo: "",
}; };
const savedConfAdminPrefs = loadFilterPrefs(confAdminPrefsKey) || {}; const savedConfAdminPrefs = loadFilterPrefs(confAdminPrefsKey) || {};
$scope.query = Object.assign({}, confAdminDefaults, savedConfAdminPrefs, { $scope.query = Object.assign({}, confAdminDefaults, savedConfAdminPrefs, {
@@ -368,15 +686,28 @@ angular
search: "", search: "",
}); });
$scope.exportCsv = () => {
const params = new URLSearchParams(
Object.entries($scope.query).filter(([, v]) => v !== "" && v !== false && v != null)
);
params.set("format", "csv");
params.set("limit", "10000");
window.open("/api/admin/conferences?" + params.toString(), "_blank");
};
$scope.fetchError = null;
function getConferences() { function getConferences() {
$scope.fetchError = null;
$http.get("/api/admin/conferences", { params: $scope.query }).then( $http.get("/api/admin/conferences", { params: $scope.query }).then(
(res) => { (res) => {
$scope.total = res.data.total; $scope.total = res.data.total;
$scope.totalPage = Math.ceil(res.data.total / $scope.query.limit); $scope.totalPage = Math.ceil(res.data.total / $scope.query.limit);
$scope.conferences = res.data.results; $scope.conferences = res.data.results;
$scope.statusCounts = res.data.statusCounts || [];
$scope.$apply(); $scope.$apply();
}, },
(err) => { (err) => {
$scope.fetchError = (err && err.data && err.data.error) || "Failed to load conferences";
console.error(err); console.error(err);
} }
); );
@@ -391,16 +722,19 @@ angular
timeClear = setTimeout(getConferences, 500); timeClear = setTimeout(getConferences, 500);
const { page, search, ...persisted } = $scope.query; const { page, search, ...persisted } = $scope.query;
saveFilterPrefs(confAdminPrefsKey, persisted); saveFilterPrefs(confAdminPrefsKey, persisted);
recomputeChipsConf();
}, },
true true
); );
recomputeChipsConf();
}, },
]) ])
.controller("queuesAdminController", [ .controller("queuesAdminController", [
"$scope", "$scope",
"$http", "$http",
"$location", "$location",
function ($scope, $http, $location) { "$interval",
function ($scope, $http, $location, $interval) {
$scope.$watch("user.status", () => { $scope.$watch("user.status", () => {
if ($scope.user == null) { if ($scope.user == null) {
$location.url("/"); $location.url("/");
@@ -412,6 +746,45 @@ angular
$scope.downloadJobs = []; $scope.downloadJobs = [];
$scope.removeJobs = []; $scope.removeJobs = [];
$scope.removeCaches = [];
$scope.counts = { download: {}, remove: {}, cache: {} };
$scope.query = {
search: "",
state: "",
autoRefresh: true,
};
$scope.jobMatchesState = (job) => {
if (!$scope.query.state) return true;
const finished = !!job.finishedOn;
const failed = (job.stacktrace || []).length > 0 || job.failedReason;
const map = {
completed: finished && !failed,
failed: failed,
active: job.processedOn && !finished,
waiting: !job.processedOn,
};
return !!map[$scope.query.state];
};
$scope.jobProgressPct = (job) => {
if (job && job.progress && typeof job.progress === "object" && typeof job.progress.percent === "number") {
return Math.max(0, Math.min(100, Math.round(job.progress.percent)));
}
if (typeof job.progress === "number") {
return Math.max(0, Math.min(100, Math.round(job.progress)));
}
return null;
};
$scope.bulkRetryFailed = (queue) => {
if (!confirm(`Retry all failed jobs in the ${queue} queue?`)) return;
$http.post(`/api/admin/queue/${queue}/retry-failed`).then(getQueues, (err) => console.error(err));
};
$scope.bulkDrain = (queue) => {
if (!confirm(`Drain (clear waiting+delayed) the ${queue} queue?`)) return;
$http.post(`/api/admin/queue/${queue}/drain`).then(getQueues, (err) => console.error(err));
};
function getQueues() { function getQueues() {
$http.get("/api/admin/queues", { params: $scope.query }).then( $http.get("/api/admin/queues", { params: $scope.query }).then(
@@ -419,6 +792,7 @@ angular
$scope.downloadJobs = res.data.downloadQueue; $scope.downloadJobs = res.data.downloadQueue;
$scope.removeJobs = res.data.removeQueue; $scope.removeJobs = res.data.removeQueue;
$scope.removeCaches = res.data.cacheQueue; $scope.removeCaches = res.data.cacheQueue;
$scope.counts = res.data.counts || $scope.counts;
}, },
(err) => { (err) => {
console.error(err); console.error(err);
@@ -427,6 +801,14 @@ angular
} }
getQueues(); getQueues();
// auto-refresh every 5 seconds while autoRefresh is on
const stop = $interval(() => {
if ($scope.query.autoRefresh) getQueues();
}, 5000);
$scope.$on("$destroy", () => $interval.cancel(stop));
$scope.refreshNow = getQueues;
$scope.removeJob = function (queue, job) { $scope.removeJob = function (queue, job) {
$http $http
.delete(`/api/admin/queue/${queue}/${job.id}`, { .delete(`/api/admin/queue/${queue}/${job.id}`, {
@@ -456,5 +838,15 @@ angular
} }
); );
}; };
let searchClear = null;
$scope.$watch(
"query.search",
() => {
clearTimeout(searchClear);
searchClear = setTimeout(getQueues, 350);
}
);
$scope.$watch("query.state", getQueues);
}, },
]); ]);
public/script/bundle.min.js
Vendored
+1 -1
View File
File diff suppressed because one or more lines are too long
src/core/Repository.ts
+12 -2
View File
@@ -22,6 +22,7 @@ import {
import { getToken } from "./GitHubUtils"; import { getToken } from "./GitHubUtils";
import config from "../config"; import config from "../config";
import FileModel from "./model/files/files.model"; import FileModel from "./model/files/files.model";
import AnonymizedRepositoryModel from "./model/anonymizedRepositories/anonymizedRepositories.model";
import { IFile } from "./model/files/files.types"; import { IFile } from "./model/files/files.types";
import AnonymizedFile from "./AnonymizedFile"; import AnonymizedFile from "./AnonymizedFile";
import { FilterQuery } from "mongoose"; import { FilterQuery } from "mongoose";
@@ -351,7 +352,7 @@ export default class Repository {
); );
await this.resetSate(RepositoryStatus.PREPARING); await this.resetSate(RepositoryStatus.PREPARING);
await downloadQueue.add(this.repoId, this, { await downloadQueue.add(this.repoId, { repoId: this.repoId }, {
jobId: this.repoId, jobId: this.repoId,
attempts: 3, attempts: 3,
}); });
@@ -405,7 +406,16 @@ export default class Repository {
this._model.statusDate = new Date(); this._model.statusDate = new Date();
this._model.statusMessage = statusMessage; this._model.statusMessage = statusMessage;
if (!isConnected) return this.model; if (!isConnected) return this.model;
await this._model.save(); await AnonymizedRepositoryModel.updateOne(
{ _id: this._model._id },
{
$set: {
status,
statusDate: this._model.statusDate,
statusMessage,
},
}
).exec();
} }
/** /**
src/core/User.ts
+30 -23
View File
@@ -80,37 +80,44 @@ export default class User {
}); });
}); });
// find the repositories that are already in the database // find the repositories that are already in the database — fetch both
const finds = ( // externalId and id so we can both detect duplicates and reuse the
await RepositoryModel.find({ // ids of existing rows without re-querying.
externalId: { const externalIds = repositories.map((repo) => repo.externalId);
$in: repositories.map((repo) => repo.externalId), const existing = await RepositoryModel.find({
}, externalId: { $in: externalIds },
}).select("externalId") }).select("id externalId");
).map((m) => m.externalId); const existingByExternalId = new Map(
existing.map((m) => [m.externalId, m.id])
// save all the new repositories
await Promise.all(
repositories
.filter((r) => finds.indexOf(r.externalId) == -1)
.map((r) => r.save())
); );
// save only the if of the repositories in the user model // save all the new repositories
this._model.repositories = ( const newRepos = repositories.filter(
await RepositoryModel.find({ (r) => !existingByExternalId.has(r.externalId)
externalId: { );
$in: repositories.map((repo) => repo.externalId), const saved = await Promise.all(newRepos.map((r) => r.save()));
}, for (const m of saved) {
}).select("id") existingByExternalId.set(m.externalId, m.id);
).map((m) => m.id); }
// collect ids in the order of the upstream repositories list
this._model.repositories = externalIds
.map((eid) => existingByExternalId.get(eid))
.filter((id) => !!id) as unknown as typeof this._model.repositories;
// have the model // have the model
await this._model.save(); await this._model.save();
return repositories.map((r) => new GitHubRepository(r)); return repositories.map((r) => new GitHubRepository(r));
} else { } else {
// Only the fields read by GitHubRepository.toJSON() (and the immediate
// callers in user routes). Branches/readme are loaded on demand by
// GitHubRepository methods, which issue their own queries.
const out = ( const out = (
await RepositoryModel.find({ _id: { $in: this._model.repositories } }) await RepositoryModel.find({
_id: { $in: this._model.repositories },
}).select(
"externalId name url size hasPage pageSource defaultBranch"
)
).map((i) => new GitHubRepository(i)); ).map((i) => new GitHubRepository(i));
return out; return out;
} }
src/core/anonymize-utils.ts
+106 -99
View File
@@ -192,8 +192,62 @@ export class AnonymizeTransformer extends Transform {
} }
} }
// Markdown image pattern hoisted out of removeImage() so we don't recompile
// it on every chunk of every file streamed through the anonymizer.
const markdownImageRegex =
/!\[[^\]]*\]\((?<filename>.*?)(?="|\))(?<optionalpart>".*")?\)/g;
interface CompiledTermVariant {
// Global regex used to replace matches in content (and paths).
replaceRegex: RegExp;
// Non-global twin used inside the URL callback to test() without
// mutating shared lastIndex state.
testRegex: RegExp;
mask: string;
}
function compileTerms(terms: string[] | undefined): CompiledTermVariant[] {
if (!terms || terms.length === 0) return [];
const compiled: CompiledTermVariant[] = [];
for (let i = 0; i < terms.length; i++) {
const spec = terms[i];
if (spec.trim() === "") continue;
// #285 — entries of the form "term=>replacement" override the default
// XXXX-N mask so users can scrub with their preferred token.
const parsed = parseTermSpec(spec);
let term = parsed.term;
const mask =
parsed.replacement !== null
? parsed.replacement
: config.ANONYMIZATION_MASK + "-" + (i + 1);
try {
new RegExp(term, "gi");
} catch {
term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
}
for (const variant of termVariants(term)) {
const bounded = withWordBoundaries(variant.pattern, {
sniffSource: variant.sniff,
unicode: variant.unicode,
});
const baseFlags = variant.unicode ? "iu" : "i";
compiled.push({
replaceRegex: new RegExp(bounded, "g" + baseFlags),
testRegex: new RegExp(bounded, baseFlags),
mask,
});
}
}
return compiled;
}
export class ContentAnonimizer { export class ContentAnonimizer {
public wasAnonymized = false; public wasAnonymized = false;
// Compiled once per instance and reused for every anonymize() call.
// Streamed files invoke anonymize() many times per file (one per chunk),
// so caching here avoids rebuilding regexes on every chunk.
private compiledTerms: CompiledTermVariant[];
private selfLinkRegexes: RegExp[] | null = null;
constructor( constructor(
readonly opt: { readonly opt: {
@@ -204,26 +258,33 @@ export class ContentAnonimizer {
branchName?: string; branchName?: string;
repoId?: string; repoId?: string;
} }
) {} ) {
this.compiledTerms = compileTerms(opt.terms);
if (opt.repoName && opt.branchName) {
const r = opt.repoName;
const b = opt.branchName;
this.selfLinkRegexes = [
new RegExp(`https://raw.githubusercontent.com/${r}/${b}\\b`, "gi"),
new RegExp(`https://github.com/${r}/blob/${b}\\b`, "gi"),
new RegExp(`https://github.com/${r}/tree/${b}\\b`, "gi"),
new RegExp(`https://github.com/${r}`, "gi"),
];
}
}
private removeImage(content: string): string { private removeImage(content: string): string {
if (this.opt.image !== false) { if (this.opt.image !== false) {
return content; return content;
} }
// remove image in markdown return content.replace(markdownImageRegex, () => {
return content.replace( this.wasAnonymized = true;
/!\[[^\]]*\]\((?<filename>.*?)(?="|\))(?<optionalpart>".*")?\)/g, return config.ANONYMIZATION_MASK;
() => { });
this.wasAnonymized = true;
return config.ANONYMIZATION_MASK;
}
);
} }
private removeLink(content: string): string { private removeLink(content: string): string {
if (this.opt.link !== false) { if (this.opt.link !== false) {
return content; return content;
} }
// remove image in markdown
return content.replace(urlRegex, () => { return content.replace(urlRegex, () => {
this.wasAnonymized = true; this.wasAnonymized = true;
return config.ANONYMIZATION_MASK; return config.ANONYMIZATION_MASK;
@@ -231,83 +292,33 @@ export class ContentAnonimizer {
} }
private replaceGitHubSelfLinks(content: string): string { private replaceGitHubSelfLinks(content: string): string {
if (!this.opt.repoName || !this.opt.branchName) { if (!this.selfLinkRegexes) return content;
return content; const replacement = `https://${config.APP_HOSTNAME}/r/${this.opt.repoId}`;
} const cb = () => {
const repoName = this.opt.repoName;
const branchName = this.opt.branchName;
const replaceCallback = () => {
this.wasAnonymized = true; this.wasAnonymized = true;
return `https://${config.APP_HOSTNAME}/r/${this.opt.repoId}`; return replacement;
}; };
content = content.replace( for (const re of this.selfLinkRegexes) {
new RegExp( content = content.replace(re, cb);
`https://raw.githubusercontent.com/${repoName}/${branchName}\\b`, }
"gi" return content;
),
replaceCallback
);
content = content.replace(
new RegExp(`https://github.com/${repoName}/blob/${branchName}\\b`, "gi"),
replaceCallback
);
content = content.replace(
new RegExp(`https://github.com/${repoName}/tree/${branchName}\\b`, "gi"),
replaceCallback
);
return content.replace(
new RegExp(`https://github.com/${repoName}`, "gi"),
replaceCallback
);
} }
private replaceTerms(content: string): string { private replaceTerms(content: string): string {
const terms = this.opt.terms || []; for (const c of this.compiledTerms) {
for (let i = 0; i < terms.length; i++) { // remove whole url if it contains the term
const spec = terms[i]; content = content.replace(urlRegex, (match) => {
if (spec.trim() == "") { if (c.testRegex.test(match)) {
continue;
}
// #285 — entries of the form "term=>replacement" override the default
// XXXX-N mask so users can scrub with their preferred token (e.g.
// "ABC", "XYZ"), keeping anonymized identifiers valid in source code.
const parsed = parseTermSpec(spec);
let term = parsed.term;
const mask =
parsed.replacement !== null
? parsed.replacement
: config.ANONYMIZATION_MASK + "-" + (i + 1);
try {
new RegExp(term, "gi");
} catch {
// escape regex characters
term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
}
// Try the term verbatim first, then a diacritic-insensitive expansion
// so "Davo" anonymizes "Davó" (and vice versa). See term-matching.ts.
for (const variant of termVariants(term)) {
const bounded = withWordBoundaries(variant.pattern, {
sniffSource: variant.sniff,
unicode: variant.unicode,
});
const flags = variant.unicode ? "giu" : "gi";
// remove whole url if it contains the term
content = content.replace(urlRegex, (match) => {
if (new RegExp(bounded, flags).test(match)) {
this.wasAnonymized = true;
return mask;
}
return match;
});
// remove the term in the text
content = content.replace(new RegExp(bounded, flags), () => {
this.wasAnonymized = true; this.wasAnonymized = true;
return mask; return c.mask;
}); }
} return match;
});
// remove the term in the text
content = content.replace(c.replaceRegex, () => {
this.wasAnonymized = true;
return c.mask;
});
} }
return content; return content;
} }
@@ -322,24 +333,20 @@ export class ContentAnonimizer {
} }
export function anonymizePath(path: string, terms: string[]) { export function anonymizePath(path: string, terms: string[]) {
for (let i = 0; i < terms.length; i++) { return anonymizePathCompiled(path, compileTerms(terms));
const spec = terms[i]; }
if (spec.trim() == "") {
continue; // Variant that accepts pre-compiled term regexes — call sites that anonymize
} // many paths in a row (tree traversal) should compile once and reuse.
const parsed = parseTermSpec(spec); export function anonymizePathCompiled(
let term = parsed.term; path: string,
const mask = compiled: CompiledTermVariant[]
parsed.replacement !== null ) {
? parsed.replacement for (const c of compiled) {
: config.ANONYMIZATION_MASK + "-" + (i + 1); path = path.replace(c.replaceRegex, c.mask);
try {
new RegExp(term, "gi");
} catch {
// escape regex characters
term = term.replace(/[-[\]{}()*+?.,\\^$|#]/g, "\\$&");
}
path = path.replace(new RegExp(term, "gi"), mask);
} }
return path; return path;
} }
export { compileTerms };
export type { CompiledTermVariant };
src/queue/index.ts
+90 -12
View File
@@ -1,11 +1,76 @@
import { Queue, Worker } from "bullmq"; import { Queue, Worker } from "bullmq";
import config from "../config"; import config from "../config";
import Repository from "../core/Repository"; import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
import { RepositoryStatus } from "../core/types";
import * as path from "path"; import * as path from "path";
export let cacheQueue: Queue<Repository>; // Minimal payload for queue jobs. Workers re-fetch the Repository from the
export let removeQueue: Queue<Repository>; // database via getRepository(repoId), so passing the full Mongoose-backed
export let downloadQueue: Queue<Repository>; // Repository instance through msgpackr is unnecessary — and triggers
// ERR_BUFFER_OUT_OF_BOUNDS on long term lists / large nested fields.
export interface RepoJobData {
repoId: string;
}
const IN_FLIGHT_STATUSES: RepositoryStatus[] = [
RepositoryStatus.PREPARING,
RepositoryStatus.QUEUE,
RepositoryStatus.DOWNLOAD,
];
async function markErrorIfInFlight(repoId: string, message: string) {
try {
await AnonymizedRepositoryModel.updateOne(
{ repoId, status: { $in: IN_FLIGHT_STATUSES } },
{
$set: {
status: RepositoryStatus.ERROR,
statusDate: new Date(),
statusMessage: message || "preparation_failed",
},
}
).exec();
} catch (e) {
console.log("[QUEUE] markErrorIfInFlight error", repoId, e);
}
}
/**
* Recover repositories left in an in-flight status (preparing/queue/download)
* with no live BullMQ job — typically caused by a worker process crash or
* server restart during anonymization. Marks them as ERROR so they don't
* appear stuck forever; the public route can re-queue them on next visit.
*/
export async function recoverStuckPreparing() {
if (!downloadQueue) return;
try {
const stuck = await AnonymizedRepositoryModel.find(
{ status: { $in: IN_FLIGHT_STATUSES } },
{ repoId: 1 }
).lean();
for (const doc of stuck) {
try {
const job = await downloadQueue.getJob(doc.repoId);
if (job) {
const state = await job.getState();
if (state === "active" || state === "waiting" || state === "delayed") {
continue;
}
}
await markErrorIfInFlight(doc.repoId, "preparation_interrupted");
console.log("[QUEUE] recovered stuck repo", doc.repoId);
} catch (e) {
console.log("[QUEUE] recover error for", doc.repoId, e);
}
}
} catch (e) {
console.log("[QUEUE] recoverStuckPreparing failed", e);
}
}
export let cacheQueue: Queue<RepoJobData>;
export let removeQueue: Queue<RepoJobData>;
export let downloadQueue: Queue<RepoJobData>;
// avoid to load the queue outside the main server // avoid to load the queue outside the main server
export function startWorker() { export function startWorker() {
@@ -14,28 +79,31 @@ export function startWorker() {
port: config.REDIS_PORT, port: config.REDIS_PORT,
}; };
cacheQueue = new Queue<Repository>("cache removal", { cacheQueue = new Queue<RepoJobData>("cache removal", {
connection, connection,
defaultJobOptions: { defaultJobOptions: {
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true,
}, },
}); });
removeQueue = new Queue<Repository>("repository removal", { removeQueue = new Queue<RepoJobData>("repository removal", {
connection: { connection: {
host: config.REDIS_HOSTNAME, host: config.REDIS_HOSTNAME,
port: config.REDIS_PORT, port: config.REDIS_PORT,
}, },
defaultJobOptions: { defaultJobOptions: {
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true,
}, },
}); });
downloadQueue = new Queue<Repository>("repository download", { downloadQueue = new Queue<RepoJobData>("repository download", {
connection, connection,
defaultJobOptions: { defaultJobOptions: {
removeOnComplete: true, removeOnComplete: true,
removeOnFail: true,
}, },
}); });
const cacheWorker = new Worker<Repository>( const cacheWorker = new Worker<RepoJobData>(
cacheQueue.name, cacheQueue.name,
path.resolve("build/queue/processes/removeCache.js"), path.resolve("build/queue/processes/removeCache.js"),
{ {
@@ -47,7 +115,7 @@ export function startWorker() {
cacheWorker.on("completed", async (job) => { cacheWorker.on("completed", async (job) => {
await job.remove(); await job.remove();
}); });
const removeWorker = new Worker<Repository>( const removeWorker = new Worker<RepoJobData>(
removeQueue.name, removeQueue.name,
path.resolve("build/queue/processes/removeRepository.js"), path.resolve("build/queue/processes/removeRepository.js"),
{ {
@@ -60,7 +128,7 @@ export function startWorker() {
await job.remove(); await job.remove();
}); });
const downloadWorker = new Worker<Repository>( const downloadWorker = new Worker<RepoJobData>(
downloadQueue.name, downloadQueue.name,
path.resolve("build/queue/processes/downloadRepository.js"), path.resolve("build/queue/processes/downloadRepository.js"),
{ {
@@ -77,7 +145,17 @@ export function startWorker() {
downloadWorker.on("completed", async (job) => { downloadWorker.on("completed", async (job) => {
console.log("[QUEUE] download repository completed", job.data.repoId); console.log("[QUEUE] download repository completed", job.data.repoId);
}); });
downloadWorker.on("failed", async (job) => { downloadWorker.on("failed", async (job, err) => {
console.log("download repository failed", job.data.repoId); const repoId = job?.data?.repoId;
console.log(
"[QUEUE] download repository failed",
repoId,
err?.message || err
);
if (!repoId) return;
if (job && typeof job.attemptsMade === "number" && job.opts?.attempts) {
if (job.attemptsMade < job.opts.attempts) return;
}
await markErrorIfInFlight(repoId, err?.message || "preparation_failed");
}); });
} }
src/queue/processes/downloadRepository.ts
+48 -27
View File
@@ -1,11 +1,11 @@
import { SandboxedJob } from "bullmq"; import { SandboxedJob } from "bullmq";
import { config } from "dotenv"; import { config } from "dotenv";
config(); config();
import Repository from "../../core/Repository";
import { getRepository as getRepositoryImport } from "../../server/database"; import { getRepository as getRepositoryImport } from "../../server/database";
import { RepositoryStatus } from "../../core/types"; import { RepositoryStatus } from "../../core/types";
import { RepoJobData } from "../index";
export default async function (job: SandboxedJob<Repository, void>) { export default async function (job: SandboxedJob<RepoJobData, void>) {
const { const {
connect, connect,
getRepository, getRepository,
@@ -18,29 +18,36 @@ export default async function (job: SandboxedJob<Repository, void>) {
let statusInterval: any = null; let statusInterval: any = null;
await connect(); await connect();
const repo = await getRepository(job.data.repoId); const repo = await getRepository(job.data.repoId);
let tickPromise: Promise<void> | null = null;
try { try {
let progress: { status: string } | null = null; let progress: { status: string } | null = null;
statusInterval = setInterval(async () => { statusInterval = setInterval(() => {
try { if (tickPromise) return;
if ( tickPromise = (async () => {
repo.status == RepositoryStatus.READY || try {
repo.status == RepositoryStatus.ERROR if (
) { repo.status == RepositoryStatus.READY ||
return clearInterval(statusInterval); repo.status == RepositoryStatus.ERROR
) {
clearInterval(statusInterval);
return;
}
if (
progress &&
repo.status &&
repo.model.statusMessage !== progress?.status
) {
console.log(
`[QUEUE] Progress: ${job.data.repoId} ${progress.status}`
);
await repo.updateStatus(repo.status, progress?.status || "");
}
} catch {
// ignore error
} finally {
tickPromise = null;
} }
if ( })();
progress &&
repo.status &&
repo.model.statusMessage !== progress?.status
) {
console.log(
`[QUEUE] Progress: ${job.data.repoId} ${progress.status}`
);
await repo.updateStatus(repo.status, progress?.status || "");
}
} catch {
// ignore error
}
}, 1000); }, 1000);
function updateProgress(obj: { status: string } | string) { function updateProgress(obj: { status: string } | string) {
const o = typeof obj === "string" ? { status: obj } : obj; const o = typeof obj === "string" ? { status: obj } : obj;
@@ -51,9 +58,12 @@ export default async function (job: SandboxedJob<Repository, void>) {
await repo.resetSate(RepositoryStatus.PREPARING, ""); await repo.resetSate(RepositoryStatus.PREPARING, "");
await repo.anonymize(updateProgress); await repo.anonymize(updateProgress);
clearInterval(statusInterval); clearInterval(statusInterval);
if (tickPromise) await tickPromise;
await repo.updateStatus(RepositoryStatus.READY, ""); await repo.updateStatus(RepositoryStatus.READY, "");
console.log(`[QUEUE] ${job.data.repoId} is downloaded`); console.log(`[QUEUE] ${job.data.repoId} is downloaded`);
} catch (error) { } catch (error) {
clearInterval(statusInterval);
if (tickPromise) await tickPromise;
updateProgress({ status: "error" }); updateProgress({ status: "error" });
if (error instanceof Error) { if (error instanceof Error) {
await repo.updateStatus(RepositoryStatus.ERROR, error.message); await repo.updateStatus(RepositoryStatus.ERROR, error.message);
@@ -64,13 +74,24 @@ export default async function (job: SandboxedJob<Repository, void>) {
} }
} catch (error: unknown) { } catch (error: unknown) {
clearInterval(statusInterval); clearInterval(statusInterval);
console.log(`[QUEUE] ${job.data.repoId} is finished with an error`, error); if (tickPromise) {
setTimeout(async () => {
// delay to avoid double saving
try { try {
await repo.updateStatus(RepositoryStatus.ERROR, (error as Error).message); await tickPromise;
} catch { /* ignored */ } } catch { /* ignored */ }
}, 400); }
console.log(`[QUEUE] ${job.data.repoId} is finished with an error`, error);
try {
await repo.updateStatus(
RepositoryStatus.ERROR,
error instanceof Error ? error.message : String(error)
);
} catch (persistError) {
console.log(
`[QUEUE] failed to persist ERROR status for ${job.data.repoId}`,
persistError
);
}
throw error;
} finally { } finally {
clearInterval(statusInterval); clearInterval(statusInterval);
} }
src/queue/processes/removeCache.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import { SandboxedJob } from "bullmq"; import { SandboxedJob } from "bullmq";
import Repository from "../../core/Repository";
import { getRepository as getRepositoryImport } from "../../server/database"; import { getRepository as getRepositoryImport } from "../../server/database";
import { RepoJobData } from "../index";
export default async function (job: SandboxedJob<Repository, void>) { export default async function (job: SandboxedJob<RepoJobData, void>) {
const { const {
connect, connect,
getRepository, getRepository,
src/queue/processes/removeRepository.ts
+2 -2
View File
@@ -1,9 +1,9 @@
import { SandboxedJob } from "bullmq"; import { SandboxedJob } from "bullmq";
import Repository from "../../core/Repository";
import { getRepository as getRepositoryImport } from "../../server/database"; import { getRepository as getRepositoryImport } from "../../server/database";
import { RepositoryStatus } from "../../core/types"; import { RepositoryStatus } from "../../core/types";
import { RepoJobData } from "../index";
export default async function (job: SandboxedJob<Repository, void>) { export default async function (job: SandboxedJob<RepoJobData, void>) {
const { const {
connect, connect,
getRepository, getRepository,
src/server/index.ts
+22 -6
View File
@@ -16,7 +16,7 @@ import { bearerTokenAuth } from "./routes/token-auth";
import router from "./routes"; import router from "./routes";
import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model"; import AnonymizedRepositoryModel from "../core/model/anonymizedRepositories/anonymizedRepositories.model";
import { conferenceStatusCheck, repositoryStatusCheck } from "./schedule"; import { conferenceStatusCheck, repositoryStatusCheck } from "./schedule";
import { startWorker } from "../queue"; import { startWorker, recoverStuckPreparing } from "../queue";
import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model"; import AnonymizedPullRequestModel from "../core/model/anonymizedPullRequests/anonymizedPullRequests.model";
import { getUser } from "./routes/route-utils"; import { getUser } from "./routes/route-utils";
import config from "../config"; import config from "../config";
@@ -165,9 +165,17 @@ export default async function start() {
apiRouter.use("/gist", speedLimiter, router.gistPrivate); apiRouter.use("/gist", speedLimiter, router.gistPrivate);
apiRouter.use("/anonymize-preview", speedLimiter, router.anonymizePreview); apiRouter.use("/anonymize-preview", speedLimiter, router.anonymizePreview);
// Cache message.txt presence so /api/message doesn't hit the filesystem
// synchronously on every request. Re-checked on a 60s interval — the file
// is admin-managed and doesn't need real-time freshness.
const messagePath = resolve("message.txt");
let messageExists = existsSync(messagePath);
setInterval(() => {
messageExists = existsSync(messagePath);
}, 60 * 1000).unref();
apiRouter.get("/message", async (_, res) => { apiRouter.get("/message", async (_, res) => {
if (existsSync("./message.txt")) { if (messageExists) {
return res.sendFile(resolve("message.txt")); return res.sendFile(messagePath);
} }
res.sendStatus(404); res.sendStatus(404);
}); });
@@ -186,10 +194,17 @@ export default async function start() {
res.json(stat); res.json(stat);
return; return;
} }
const [nbRepositories, users, nbPageViews, nbPullRequests] = const [nbRepositories, nbUsersAgg, nbPageViews, nbPullRequests] =
await Promise.all([ await Promise.all([
AnonymizedRepositoryModel.estimatedDocumentCount(), AnonymizedRepositoryModel.estimatedDocumentCount(),
AnonymizedRepositoryModel.distinct("owner"), // Count distinct owners server-side instead of materializing the full
// list of ObjectIds with `.distinct("owner")` only to take its length.
AnonymizedRepositoryModel.collection
.aggregate([
{ $group: { _id: "$owner" } },
{ $count: "n" },
])
.toArray(),
AnonymizedRepositoryModel.collection AnonymizedRepositoryModel.collection
.aggregate([ .aggregate([
{ {
@@ -202,7 +217,7 @@ export default async function start() {
stat = { stat = {
nbRepositories, nbRepositories,
nbUsers: users.length, nbUsers: (nbUsersAgg[0] as { n?: number } | undefined)?.n || 0,
nbPageViews: nbPageViews[0]?.total || 0, nbPageViews: nbPageViews[0]?.total || 0,
nbPullRequests, nbPullRequests,
}; };
@@ -235,6 +250,7 @@ export default async function start() {
repositoryStatusCheck(); repositoryStatusCheck();
await connect(); await connect();
await recoverStuckPreparing();
app.listen(config.PORT); app.listen(config.PORT);
console.log("Database connected and Server started on port: " + config.PORT); console.log("Database connected and Server started on port: " + config.PORT);
} }
src/server/routes/admin.ts
+428 -107
View File
@@ -10,6 +10,7 @@ import User from "../../core/User";
import { ensureAuthenticated } from "./connection"; import { ensureAuthenticated } from "./connection";
import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils"; import { handleError, getUser, isOwnerOrAdmin, getRepo } from "./route-utils";
import adminTokensRouter from "./admin-tokens"; import adminTokensRouter from "./admin-tokens";
import { octokit, getToken } from "../../core/GitHubUtils";
const router = express.Router(); const router = express.Router();
@@ -34,17 +35,69 @@ router.use(
router.use("/tokens", adminTokensRouter); router.use("/tokens", adminTokensRouter);
router.post("/queue/:name/:repo_id", async (req, res) => { const QUEUE_STATES = [
let queue: Queue<Repository, void>; "waiting",
if (req.params.name == "download") { "active",
queue = downloadQueue; "completed",
} else if (req.params.name == "cache") { "failed",
queue = cacheQueue; "delayed",
} else if (req.params.name == "remove") { ] as const;
queue = removeQueue;
} else { function pickQueue(name: string): Queue | null {
return res.status(404).json({ error: "queue_not_found" }); if (name === "download") return downloadQueue;
if (name === "cache") return cacheQueue;
if (name === "remove") return removeQueue;
return null;
}
function escapeRegex(s: string): string {
return s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
}
function parseSort(req: express.Request, fallbackField = "_id"): Record<string, 1 | -1> {
const direction = req.query.direction === "asc" ? 1 : -1;
const field = (req.query.sort as string) || fallbackField;
return { [field]: direction };
}
function parseDateRange(req: express.Request, field: string) {
const range: Record<string, Date> = {};
if (req.query.dateFrom) {
const d = new Date(req.query.dateFrom as string);
if (!isNaN(d.getTime())) range.$gte = d;
} }
if (req.query.dateTo) {
const d = new Date(req.query.dateTo as string);
if (!isNaN(d.getTime())) range.$lte = d;
}
if (Object.keys(range).length === 0) return null;
return { [field]: range };
}
function csvEscape(v: unknown): string {
if (v == null) return "";
const s = typeof v === "object" ? JSON.stringify(v) : String(v);
if (/[",\n\r]/.test(s)) return `"${s.replace(/"/g, '""')}"`;
return s;
}
function sendCsv(
res: express.Response,
filename: string,
columns: string[],
rows: Array<Record<string, unknown>>
) {
const header = columns.join(",");
const lines = rows.map((r) => columns.map((c) => csvEscape(r[c])).join(","));
const body = [header, ...lines].join("\n");
res.setHeader("Content-Type", "text/csv; charset=utf-8");
res.setHeader("Content-Disposition", `attachment; filename="${filename}"`);
res.send(body);
}
router.post("/queue/:name/:repo_id", async (req, res) => {
const queue = pickQueue(req.params.name);
if (!queue) return res.status(404).json({ error: "queue_not_found" });
let job; let job;
try { try {
job = await queue.getJob(req.params.repo_id); job = await queue.getJob(req.params.repo_id);
@@ -68,16 +121,8 @@ router.post("/queue/:name/:repo_id", async (req, res) => {
}); });
router.delete("/queue/:name/:repo_id", async (req, res) => { router.delete("/queue/:name/:repo_id", async (req, res) => {
let queue: Queue; const queue = pickQueue(req.params.name);
if (req.params.name == "download") { if (!queue) return res.status(404).json({ error: "queue_not_found" });
queue = downloadQueue;
} else if (req.params.name == "cache") {
queue = cacheQueue;
} else if (req.params.name == "remove") {
queue = removeQueue;
} else {
return res.status(404).json({ error: "queue_not_found" });
}
try { try {
const job = await queue.getJob(req.params.repo_id); const job = await queue.getJob(req.params.repo_id);
if (!job) { if (!job) {
@@ -90,58 +135,153 @@ router.delete("/queue/:name/:repo_id", async (req, res) => {
} }
}); });
// Bulk retry all failed in a queue
router.post("/queue/:name/retry-failed", async (req, res) => {
const queue = pickQueue(req.params.name);
if (!queue) return res.status(404).json({ error: "queue_not_found" });
try {
const failed = await queue.getJobs(["failed"]);
let count = 0;
for (const j of failed) {
try {
await j.retry();
count++;
} catch {
// ignore single job failures
}
}
res.json({ retried: count, total: failed.length });
} catch (error) {
handleError(error, res, req);
}
});
// Bulk drain all waiting/delayed
router.post("/queue/:name/drain", async (req, res) => {
const queue = pickQueue(req.params.name);
if (!queue) return res.status(404).json({ error: "queue_not_found" });
try {
await queue.drain(true);
res.json({ ok: true });
} catch (error) {
handleError(error, res, req);
}
});
router.get("/queues", async (req, res) => { router.get("/queues", async (req, res) => {
const out = await Promise.all([ const search = req.query.search ? String(req.query.search).toLowerCase() : "";
downloadQueue.getJobs([ const stateFilter = req.query.state ? String(req.query.state) : null;
"waiting", const states = stateFilter && (QUEUE_STATES as readonly string[]).includes(stateFilter)
"active", ? [stateFilter]
"completed", : (QUEUE_STATES as readonly string[]);
"failed",
"delayed", const [download, remove, cache, dCounts, rCounts, cCounts] = await Promise.all([
]), downloadQueue.getJobs(states),
removeQueue.getJobs([ removeQueue.getJobs(states),
"waiting", cacheQueue.getJobs(states),
"active", downloadQueue.getJobCounts(...QUEUE_STATES),
"completed", removeQueue.getJobCounts(...QUEUE_STATES),
"failed", cacheQueue.getJobCounts(...QUEUE_STATES),
"delayed",
]),
cacheQueue.getJobs(["waiting", "active", "completed", "failed", "delayed"]),
]); ]);
const matches = (job: { id?: string | undefined; name?: string }) => {
if (!search) return true;
return (
(job.id || "").toLowerCase().includes(search) ||
(job.name || "").toLowerCase().includes(search)
);
};
res.json({ res.json({
downloadQueue: out[0], downloadQueue: download.filter(matches),
removeQueue: out[1], removeQueue: remove.filter(matches),
cacheQueue: out[2], cacheQueue: cache.filter(matches),
counts: {
download: dCounts,
remove: rCounts,
cache: cCounts,
},
}); });
}); });
// Global stats endpoint: counts by status, total disk, recent failures
router.get("/stats", async (req, res) => {
try {
const [statusBreakdown, totalSize, recentErrors, totalUsers, totalConferences] =
await Promise.all([
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.countDocuments({
status: "error",
statusDate: { $gte: new Date(Date.now() - 1000 * 60 * 60 * 24) },
}),
UserModel.estimatedDocumentCount(),
ConferenceModel.estimatedDocumentCount(),
]);
res.json({
statusBreakdown,
totalStorage: totalSize[0]?.total || 0,
recentErrors24h: recentErrors,
totalUsers,
totalConferences,
});
} catch (error) {
handleError(error, res, req);
}
});
router.get("/repos", async (req, res) => { router.get("/repos", async (req, res) => {
const page = parseInt(req.query.page as string) || 1; const page = parseInt(req.query.page as string) || 1;
const limit = parseInt(req.query.limit as string) || 10; const limit = Math.min(parseInt(req.query.limit as string) || 10, 1000);
const ready = req.query.ready == "true"; const ready = req.query.ready == "true";
const error = req.query.error == "true"; const error = req.query.error == "true";
const preparing = req.query.preparing == "true"; const preparing = req.query.preparing == "true";
const remove = req.query.removed == "true"; const remove = req.query.removed == "true";
const expired = req.query.expired == "true"; const expired = req.query.expired == "true";
// eslint-disable-next-line @typescript-eslint/no-explicit-any const sort = parseSort(req);
let sort: any = { _id: 1 }; const query: Record<string, unknown>[] = [];
if (req.query.sort) {
sort = {}; // multi-field search: repoId, source.repositoryName, statusMessage, conference
sort[req.query.sort as string] = -1;
}
const query = [];
if (req.query.search) { if (req.query.search) {
const escaped = (req.query.search as string).replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); const escaped = escapeRegex(req.query.search as string);
query.push({ repoId: { $regex: escaped } }); const re = { $regex: escaped, $options: "i" };
query.push({
$or: [
{ repoId: re },
{ "source.repositoryName": re },
{ statusMessage: re },
{ conference: re },
],
});
} }
// filter by owner username
if (req.query.owner) {
const ownerUsername = req.query.owner as string;
const ownerDoc = await UserModel.findOne({ username: ownerUsername }, { _id: 1 });
if (!ownerDoc) {
return res.json({ query: { $and: query }, page, total: 0, sort, results: [], statusCounts: [], totalSize: 0 });
}
query.push({ owner: ownerDoc._id });
}
// filter by conference
if (req.query.conference) {
query.push({ conference: req.query.conference });
}
// date range filter on anonymizeDate
const dateFilter = parseDateRange(req, "anonymizeDate");
if (dateFilter) query.push(dateFilter);
const status: { status: string }[] = []; const status: { status: string }[] = [];
if (ready) { if (ready) status.push({ status: "ready" });
status.push({ status: "ready" }); if (error) status.push({ status: "error" });
}
if (error) {
status.push({ status: "error" });
}
if (expired) { if (expired) {
status.push({ status: "expiring" }); status.push({ status: "expiring" });
status.push({ status: "expired" }); status.push({ status: "expired" });
@@ -157,23 +297,59 @@ router.get("/repos", async (req, res) => {
if (status.length > 0) { if (status.length > 0) {
query.push({ $or: status }); query.push({ $or: status });
} }
const filter = query.length ? { $and: query } : {};
const skipIndex = (page - 1) * limit; const skipIndex = (page - 1) * limit;
const [total, results] = await Promise.all([
AnonymizedRepositoryModel.find({ // CSV export branch
$and: query, if (req.query.format === "csv") {
}).countDocuments(), const all = await AnonymizedRepositoryModel.find(filter).sort(sort).limit(50000).lean();
AnonymizedRepositoryModel.find({ $and: query }) const rows = all.map((r) => ({
repoId: r.repoId,
status: r.status,
statusMessage: r.statusMessage || "",
anonymizeDate: r.anonymizeDate ? new Date(r.anonymizeDate).toISOString() : "",
lastView: r.lastView ? new Date(r.lastView).toISOString() : "",
pageView: r.pageView || 0,
sourceRepository: r.source?.repositoryName || "",
sourceBranch: r.source?.branch || "",
sourceCommit: r.source?.commit || "",
conference: r.conference || "",
storage: r.size?.storage || 0,
terms: (r.options?.terms || []).length,
}));
return sendCsv(
res,
`repositories-${new Date().toISOString().slice(0, 10)}.csv`,
Object.keys(rows[0] || { repoId: 1 }),
rows
);
}
const [total, results, statusCounts, sizeAgg] = await Promise.all([
AnonymizedRepositoryModel.find(filter).countDocuments(),
AnonymizedRepositoryModel.find(filter)
.skip(skipIndex) .skip(skipIndex)
.sort(sort) .sort(sort)
.limit(limit) .limit(limit)
.exec(), .exec(),
AnonymizedRepositoryModel.aggregate([
{ $match: filter },
{ $group: { _id: "$status", count: { $sum: 1 }, storage: { $sum: "$size.storage" } } },
]),
AnonymizedRepositoryModel.aggregate([
{ $match: filter },
{ $group: { _id: null, total: { $sum: "$size.storage" } } },
]),
]); ]);
res.json({ res.json({
query: { $and: query }, query: filter,
page, page,
total, total,
sort, sort,
results, results,
statusCounts,
totalSize: sizeAgg[0]?.total || 0,
}); });
}); });
@@ -184,7 +360,7 @@ router.delete(
const repo = await getRepo(req, res, { nocheck: true }); const repo = await getRepo(req, res, { nocheck: true });
if (!repo) return; if (!repo) return;
try { try {
await cacheQueue.add(repo.repoId, repo, { jobId: repo.repoId }); await cacheQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: repo.repoId });
return res.json({ status: repo.status }); return res.json({ status: repo.status });
} catch (error) { } catch (error) {
handleError(error, res, req); handleError(error, res, req);
@@ -192,33 +368,163 @@ router.delete(
} }
); );
// Live GitHub info for a repository (admin diagnostic)
router.get(
"/repos/:repoId/github",
async (req: express.Request, res: express.Response) => {
try {
const repo = await getRepo(req, res, { nocheck: true });
if (!repo) return;
let token: string | undefined;
try {
token = await getToken(repo);
} catch {
token = undefined;
}
const oct = octokit(token || "");
const fullName = repo.model.source?.repositoryName || "";
const [owner, name] = fullName.split("/");
if (!owner || !name) {
return res.status(400).json({ error: "invalid_source_repository" });
}
const out: Record<string, unknown> = {
source: { owner, repo: name, branch: repo.model.source?.branch, commit: repo.model.source?.commit },
};
try {
const info = await oct.repos.get({ owner, repo: name });
out.repository = {
fullName: info.data.full_name,
private: info.data.private,
archived: info.data.archived,
disabled: info.data.disabled,
defaultBranch: info.data.default_branch,
description: info.data.description,
stargazers: info.data.stargazers_count,
watchers: info.data.watchers_count,
forks: info.data.forks_count,
openIssues: info.data.open_issues_count,
size: info.data.size,
language: info.data.language,
license: info.data.license?.spdx_id,
createdAt: info.data.created_at,
updatedAt: info.data.updated_at,
pushedAt: info.data.pushed_at,
htmlUrl: info.data.html_url,
topics: info.data.topics,
};
} catch (e) {
out.repositoryError = (e as Error)?.message || String(e);
}
try {
if (repo.model.source?.branch) {
const br = await oct.repos.getBranch({ owner, repo: name, branch: repo.model.source.branch });
out.branch = {
name: br.data.name,
protected: br.data.protected,
commitSha: br.data.commit?.sha,
};
}
} catch (e) {
out.branchError = (e as Error)?.message || String(e);
}
try {
if (repo.model.source?.commit) {
const c = await oct.repos.getCommit({ owner, repo: name, ref: repo.model.source.commit });
out.commit = {
sha: c.data.sha,
message: c.data.commit?.message,
author: c.data.commit?.author,
committer: c.data.commit?.committer,
htmlUrl: c.data.html_url,
stats: c.data.stats,
filesChanged: c.data.files?.length,
};
}
} catch (e) {
out.commitError = (e as Error)?.message || String(e);
}
try {
const r = await oct.rateLimit.get();
out.rateLimit = {
remaining: r.data.rate.remaining,
limit: r.data.rate.limit,
reset: new Date(r.data.rate.reset * 1000).toISOString(),
};
} catch {
// ignore
}
res.json(out);
} catch (error) {
handleError(error, res, req);
}
}
);
router.get("/users", async (req, res) => { router.get("/users", async (req, res) => {
const page = parseInt(req.query.page as string) || 1; const page = parseInt(req.query.page as string) || 1;
const limit = parseInt(req.query.limit as string) || 10; const limit = Math.min(parseInt(req.query.limit as string) || 10, 1000);
const skipIndex = (page - 1) * limit; const skipIndex = (page - 1) * limit;
// eslint-disable-next-line @typescript-eslint/no-explicit-any const sort = parseSort(req);
let sort: any = { _id: 1 }; const filter: Record<string, unknown> = {};
if (req.query.sort) {
sort = {};
sort[req.query.sort as string] = -1;
}
let query = {};
if (req.query.search) { if (req.query.search) {
const escaped = (req.query.search as string).replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); const escaped = escapeRegex(req.query.search as string);
query = { username: { $regex: escaped } }; filter.$or = [
{ username: { $regex: escaped, $options: "i" } },
{ "emails.email": { $regex: escaped, $options: "i" } },
];
}
if (req.query.status) {
filter.status = req.query.status;
}
if (req.query.role === "admin") {
filter.isAdmin = true;
}
const dateFilter = parseDateRange(req, "dateOfEntry");
if (dateFilter) Object.assign(filter, dateFilter);
// CSV export
if (req.query.format === "csv") {
const all = await UserModel.find(filter).sort(sort).limit(50000).lean();
const rows = all.map((u) => ({
username: u.username,
email: u.emails?.[0]?.email || "",
status: u.status,
isAdmin: !!u.isAdmin,
repoCount: (u.repositories || []).length,
dateOfEntry: u.dateOfEntry ? new Date(u.dateOfEntry).toISOString() : "",
}));
return sendCsv(
res,
`users-${new Date().toISOString().slice(0, 10)}.csv`,
["username", "email", "status", "isAdmin", "repoCount", "dateOfEntry"],
rows
);
} }
res.json({ const [total, results, statusCounts] = await Promise.all([
query: query, UserModel.find(filter).countDocuments(),
page, UserModel.aggregate([
total: await UserModel.find(query).countDocuments(), { $match: filter },
sort, { $sort: sort },
results: await UserModel.find(query) { $skip: skipIndex },
.sort(sort) { $limit: limit },
.limit(limit) {
.skip(skipIndex), $addFields: {
}); repoCount: { $size: { $ifNull: ["$repositories", []] } },
},
},
{ $project: { accessTokens: 0, apiTokens: 0 } },
]),
UserModel.aggregate([
{ $match: filter },
{ $group: { _id: "$status", count: { $sum: 1 } } },
]),
]);
res.json({ query: filter, page, total, sort, results, statusCounts });
}); });
router.get( router.get(
"/users/:username", "/users/:username",
@@ -266,35 +572,50 @@ router.get(
); );
router.get("/conferences", async (req, res) => { router.get("/conferences", async (req, res) => {
const page = parseInt(req.query.page as string) || 1; const page = parseInt(req.query.page as string) || 1;
const limit = parseInt(req.query.limit as string) || 10; const limit = Math.min(parseInt(req.query.limit as string) || 10, 1000);
const skipIndex = (page - 1) * limit; const skipIndex = (page - 1) * limit;
// eslint-disable-next-line @typescript-eslint/no-explicit-any const sort = parseSort(req);
let sort: any = { _id: 1 }; const filter: Record<string, unknown> = {};
if (req.query.sort) {
sort = {};
sort[req.query.sort as string] = -1;
}
let query = {};
if (req.query.search) { if (req.query.search) {
const escaped = (req.query.search as string).replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&"); const escaped = escapeRegex(req.query.search as string);
query = { filter.$or = [
$or: [ { name: { $regex: escaped, $options: "i" } },
{ name: { $regex: escaped } }, { conferenceID: { $regex: escaped, $options: "i" } },
{ conferenceID: { $regex: escaped } }, ];
],
};
} }
res.json({ if (req.query.status) filter.status = req.query.status;
query: query, const dateFilter = parseDateRange(req, "startDate");
page, if (dateFilter) Object.assign(filter, dateFilter);
total: await ConferenceModel.find(query).estimatedDocumentCount(),
sort, if (req.query.format === "csv") {
results: await ConferenceModel.find(query) const all = await ConferenceModel.find(filter).sort(sort).limit(50000).lean();
.sort(sort) const rows = all.map((c: Record<string, unknown>) => ({
.limit(limit) conferenceID: c.conferenceID,
.skip(skipIndex), name: c.name,
}); status: c.status,
price: c.price || 0,
repoCount: ((c.repositories as unknown[]) || []).length,
startDate: c.startDate ? new Date(c.startDate as Date).toISOString() : "",
endDate: c.endDate ? new Date(c.endDate as Date).toISOString() : "",
}));
return sendCsv(
res,
`conferences-${new Date().toISOString().slice(0, 10)}.csv`,
["conferenceID", "name", "status", "price", "repoCount", "startDate", "endDate"],
rows
);
}
const [total, results, statusCounts] = await Promise.all([
ConferenceModel.find(filter).countDocuments(),
ConferenceModel.find(filter).sort(sort).limit(limit).skip(skipIndex),
ConferenceModel.aggregate([
{ $match: filter },
{ $group: { _id: "$status", count: { $sum: 1 } } },
]),
]);
res.json({ query: filter, page, total, sort, results, statusCounts });
}); });
export default router; export default router;
src/server/routes/repository-private.ts
+3 -3
View File
@@ -175,7 +175,7 @@ router.delete(
const user = await getUser(req); const user = await getUser(req);
isOwnerOrAdmin([repo.owner.id], user); isOwnerOrAdmin([repo.owner.id], user);
await repo.updateStatus(RepositoryStatus.REMOVING); await repo.updateStatus(RepositoryStatus.REMOVING);
await removeQueue.add(repo.repoId, repo, { jobId: repo.repoId }); await removeQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: repo.repoId });
return res.json({ status: repo.status }); return res.json({ status: repo.status });
} catch (error) { } catch (error) {
handleError(error, res, req); handleError(error, res, req);
@@ -470,7 +470,7 @@ router.post(
repo.model.conference = repoUpdate.conference; repo.model.conference = repoUpdate.conference;
await repo.updateStatus(RepositoryStatus.PREPARING); await repo.updateStatus(RepositoryStatus.PREPARING);
res.json({ status: repo.status }); res.json({ status: repo.status });
await downloadQueue.add(repo.repoId, repo, { jobId: repo.repoId }); await downloadQueue.add(repo.repoId, { repoId: repo.repoId }, { jobId: repo.repoId });
} catch (error) { } catch (error) {
return handleError(error, res, req); return handleError(error, res, req);
} }
@@ -559,7 +559,7 @@ router.post("/", async (req: express.Request, res: express.Response) => {
} }
res.send({ status: repo.status }); res.send({ status: repo.status });
downloadQueue.add(repo.repoId, new Repository(repo), { downloadQueue.add(repo.repoId, { repoId: repo.repoId }, {
jobId: repo.repoId, jobId: repo.repoId,
attempts: 3, attempts: 3,
}); });
src/server/routes/repository-public.ts
+1 -1
View File
@@ -178,7 +178,7 @@ router.get(
// && repo.status != "preparing" // && repo.status != "preparing"
) { ) {
await repo.updateStatus(RepositoryStatus.PREPARING); await repo.updateStatus(RepositoryStatus.PREPARING);
await downloadQueue.add(repo.repoId, repo, { await downloadQueue.add(repo.repoId, { repoId: repo.repoId }, {
jobId: repo.repoId, jobId: repo.repoId,
attempts: 3, attempts: 3,
}); });
src/server/routes/user.ts
+3 -2
View File
@@ -61,6 +61,7 @@ router.get("/quota", async (req: express.Request, res: express.Response) => {
} }
if (uncachedIds.length) { if (uncachedIds.length) {
const uncachedSet = new Set(uncachedIds);
const agg = await FileModel.aggregate([ const agg = await FileModel.aggregate([
{ $match: { repoId: { $in: uncachedIds } } }, { $match: { repoId: { $in: uncachedIds } } },
{ {
@@ -76,7 +77,7 @@ router.get("/quota", async (req: express.Request, res: express.Response) => {
byId.set(row._id, { storage: row.storage || 0, file: row.file || 0 }); byId.set(row._id, { storage: row.storage || 0, file: row.file || 0 });
} }
for (const r of ready) { for (const r of ready) {
if (!uncachedIds.includes(r.repoId)) continue; if (!uncachedSet.has(r.repoId)) continue;
const size = byId.get(r.repoId) || { storage: 0, file: 0 }; const size = byId.get(r.repoId) || { storage: 0, file: 0 };
totalStorage += size.storage; totalStorage += size.storage;
totalFiles += size.file; totalFiles += size.file;
@@ -85,7 +86,7 @@ router.get("/quota", async (req: express.Request, res: express.Response) => {
if (isConnected) { if (isConnected) {
await Promise.all( await Promise.all(
ready ready
.filter((r) => uncachedIds.includes(r.repoId)) .filter((r) => uncachedSet.has(r.repoId))
.map((r) => r.model.save()) .map((r) => r.model.save())
); );
} }