mirror of
https://github.com/OutlineFoundation/outline-server.git
synced 2026-08-04 14:37:34 +00:00
feat(server): add tunnel time metric to opt-in server usage report (#1551)
* feat(server): add ASN metric to opt-in server usage report * feat(server): add tunnel time metric to opt-in server usage report * Rename variable. * Let Prometheus join the metrics. * Use a Map. * Revert changes to `prometheus_scraper.ts`. * Rename `LocationUsage` to `ReportedUsage`. * Add test cases for different ASN+country combinations. * Use a different approach where we don't let Prometheus combine the series.
This commit is contained in:
Sander Bruens
GitHub
parent
845c02373b
commit
e416938950
3 changed files with 80 additions and 49 deletions
|
|
@ -59,7 +59,7 @@ export class FakePrometheusClient extends PrometheusClient {
|
|||
const bytesTransferred = this.bytesTransferredById[accessKeyId] || 0;
|
||||
queryResultData.result.push({
|
||||
metric: {access_key: accessKeyId},
|
||||
value: [bytesTransferred, `${bytesTransferred}`],
|
||||
value: [Date.now() / 1000, `${bytesTransferred}`],
|
||||
});
|
||||
}
|
||||
return queryResultData;
|
||||
|
|
|
|||
|
|
@ -79,11 +79,11 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
|
||||
publisher.startSharing();
|
||||
usageMetrics.reportedUsage = [
|
||||
{country: 'AA', inboundBytes: 11},
|
||||
{country: 'BB', inboundBytes: 11},
|
||||
{country: 'CC', inboundBytes: 22},
|
||||
{country: 'AA', inboundBytes: 33},
|
||||
{country: 'DD', inboundBytes: 33},
|
||||
{country: 'AA', inboundBytes: 11, tunnelTimeSec: 99},
|
||||
{country: 'BB', inboundBytes: 11, tunnelTimeSec: 88},
|
||||
{country: 'CC', inboundBytes: 22, tunnelTimeSec: 77},
|
||||
{country: 'AA', inboundBytes: 33, tunnelTimeSec: 66},
|
||||
{country: 'DD', inboundBytes: 33, tunnelTimeSec: 55},
|
||||
];
|
||||
|
||||
clock.nowMs += 60 * 60 * 1000;
|
||||
|
|
@ -93,18 +93,18 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
startUtcMs: startTime,
|
||||
endUtcMs: clock.nowMs,
|
||||
userReports: [
|
||||
{bytesTransferred: 11, countries: ['AA']},
|
||||
{bytesTransferred: 11, countries: ['BB']},
|
||||
{bytesTransferred: 22, countries: ['CC']},
|
||||
{bytesTransferred: 33, countries: ['AA']},
|
||||
{bytesTransferred: 33, countries: ['DD']},
|
||||
{bytesTransferred: 11, countries: ['AA'], tunnelTimeSec: 99},
|
||||
{bytesTransferred: 11, countries: ['BB'], tunnelTimeSec: 88},
|
||||
{bytesTransferred: 22, countries: ['CC'], tunnelTimeSec: 77},
|
||||
{bytesTransferred: 33, countries: ['AA'], tunnelTimeSec: 66},
|
||||
{bytesTransferred: 33, countries: ['DD'], tunnelTimeSec: 55},
|
||||
],
|
||||
});
|
||||
|
||||
startTime = clock.nowMs;
|
||||
usageMetrics.reportedUsage = [
|
||||
{country: 'EE', inboundBytes: 44},
|
||||
{country: 'FF', inboundBytes: 55},
|
||||
{country: 'EE', inboundBytes: 44, tunnelTimeSec: 11},
|
||||
{country: 'FF', inboundBytes: 55, tunnelTimeSec: 22},
|
||||
];
|
||||
|
||||
clock.nowMs += 60 * 60 * 1000;
|
||||
|
|
@ -114,8 +114,8 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
startUtcMs: startTime,
|
||||
endUtcMs: clock.nowMs,
|
||||
userReports: [
|
||||
{bytesTransferred: 44, countries: ['EE']},
|
||||
{bytesTransferred: 55, countries: ['FF']},
|
||||
{bytesTransferred: 44, countries: ['EE'], tunnelTimeSec: 11},
|
||||
{bytesTransferred: 55, countries: ['FF'], tunnelTimeSec: 22},
|
||||
],
|
||||
});
|
||||
|
||||
|
|
@ -137,15 +137,15 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
publisher.startSharing();
|
||||
|
||||
usageMetrics.reportedUsage = [
|
||||
{country: 'DD', asn: 999, inboundBytes: 44},
|
||||
{country: 'EE', inboundBytes: 55},
|
||||
{country: 'DD', inboundBytes: 44, tunnelTimeSec: 11, asn: 999},
|
||||
{country: 'EE', inboundBytes: 55, tunnelTimeSec: 22},
|
||||
];
|
||||
clock.nowMs += 60 * 60 * 1000;
|
||||
await clock.runCallbacks();
|
||||
|
||||
expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([
|
||||
{bytesTransferred: 44, countries: ['DD'], asn: 999},
|
||||
{bytesTransferred: 55, countries: ['EE']},
|
||||
{bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999},
|
||||
{bytesTransferred: 55, tunnelTimeSec: 22, countries: ['EE']},
|
||||
]);
|
||||
publisher.stopSharing();
|
||||
});
|
||||
|
|
@ -165,15 +165,15 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
publisher.startSharing();
|
||||
|
||||
usageMetrics.reportedUsage = [
|
||||
{country: 'DD', asn: 999, inboundBytes: 44},
|
||||
{country: 'DD', asn: 888, inboundBytes: 55},
|
||||
{country: 'DD', asn: 999, tunnelTimeSec: 11, inboundBytes: 44},
|
||||
{country: 'DD', asn: 888, tunnelTimeSec: 22, inboundBytes: 55},
|
||||
];
|
||||
clock.nowMs += 60 * 60 * 1000;
|
||||
await clock.runCallbacks();
|
||||
|
||||
expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([
|
||||
{bytesTransferred: 44, countries: ['DD'], asn: 999},
|
||||
{bytesTransferred: 55, countries: ['DD'], asn: 888},
|
||||
{bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999},
|
||||
{bytesTransferred: 55, tunnelTimeSec: 22, countries: ['DD'], asn: 888},
|
||||
]);
|
||||
publisher.stopSharing();
|
||||
});
|
||||
|
|
@ -193,15 +193,15 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
publisher.startSharing();
|
||||
|
||||
usageMetrics.reportedUsage = [
|
||||
{country: 'DD', asn: 999, inboundBytes: 44},
|
||||
{country: 'EE', asn: 999, inboundBytes: 66},
|
||||
{country: 'DD', asn: 999, tunnelTimeSec: 11, inboundBytes: 44},
|
||||
{country: 'EE', asn: 999, tunnelTimeSec: 22, inboundBytes: 55},
|
||||
];
|
||||
clock.nowMs += 60 * 60 * 1000;
|
||||
await clock.runCallbacks();
|
||||
|
||||
expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([
|
||||
{bytesTransferred: 44, countries: ['DD'], asn: 999},
|
||||
{bytesTransferred: 66, countries: ['EE'], asn: 999},
|
||||
{bytesTransferred: 44, tunnelTimeSec: 11, countries: ['DD'], asn: 999},
|
||||
{bytesTransferred: 55, tunnelTimeSec: 22, countries: ['EE'], asn: 999},
|
||||
]);
|
||||
publisher.stopSharing();
|
||||
});
|
||||
|
|
@ -222,11 +222,11 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
|
||||
publisher.startSharing();
|
||||
usageMetrics.reportedUsage = [
|
||||
{country: 'AA', inboundBytes: 11},
|
||||
{country: 'SY', inboundBytes: 11},
|
||||
{country: 'CC', inboundBytes: 22},
|
||||
{country: 'AA', inboundBytes: 33},
|
||||
{country: 'DD', inboundBytes: 33},
|
||||
{country: 'AA', tunnelTimeSec: 99, inboundBytes: 11},
|
||||
{country: 'SY', tunnelTimeSec: 88, inboundBytes: 11},
|
||||
{country: 'CC', tunnelTimeSec: 77, inboundBytes: 22},
|
||||
{country: 'AA', tunnelTimeSec: 66, inboundBytes: 33},
|
||||
{country: 'DD', tunnelTimeSec: 55, inboundBytes: 33},
|
||||
];
|
||||
|
||||
clock.nowMs += 60 * 60 * 1000;
|
||||
|
|
@ -236,10 +236,10 @@ describe('OutlineSharedMetricsPublisher', () => {
|
|||
startUtcMs: startTime,
|
||||
endUtcMs: clock.nowMs,
|
||||
userReports: [
|
||||
{bytesTransferred: 11, countries: ['AA']},
|
||||
{bytesTransferred: 22, countries: ['CC']},
|
||||
{bytesTransferred: 33, countries: ['AA']},
|
||||
{bytesTransferred: 33, countries: ['DD']},
|
||||
{bytesTransferred: 11, tunnelTimeSec: 99, countries: ['AA']},
|
||||
{bytesTransferred: 22, tunnelTimeSec: 77, countries: ['CC']},
|
||||
{bytesTransferred: 33, tunnelTimeSec: 66, countries: ['AA']},
|
||||
{bytesTransferred: 33, tunnelTimeSec: 55, countries: ['DD']},
|
||||
],
|
||||
});
|
||||
publisher.stopSharing();
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import {Clock} from '../infrastructure/clock';
|
|||
import * as follow_redirects from '../infrastructure/follow_redirects';
|
||||
import {JsonConfig} from '../infrastructure/json_config';
|
||||
import * as logging from '../infrastructure/logging';
|
||||
import {PrometheusClient} from '../infrastructure/prometheus_scraper';
|
||||
import {PrometheusClient, QueryResultData} from '../infrastructure/prometheus_scraper';
|
||||
import * as version from './version';
|
||||
import {AccessKeyConfigJson} from './server_access_key';
|
||||
|
||||
|
|
@ -30,6 +30,7 @@ export interface ReportedUsage {
|
|||
country: string;
|
||||
asn?: number;
|
||||
inboundBytes: number;
|
||||
tunnelTimeSec: number;
|
||||
}
|
||||
|
||||
// JSON format for the published report.
|
||||
|
|
@ -47,6 +48,7 @@ export interface HourlyUserMetricsReportJson {
|
|||
countries: string[];
|
||||
asn?: number;
|
||||
bytesTransferred: number;
|
||||
tunnelTimeSec: number;
|
||||
}
|
||||
|
||||
// JSON format for the feature metrics report.
|
||||
|
|
@ -84,18 +86,46 @@ export class PrometheusUsageMetrics implements UsageMetrics {
|
|||
|
||||
async getReportedUsage(): Promise<ReportedUsage[]> {
|
||||
const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000);
|
||||
// We measure the traffic to and from the target, since that's what we are protecting.
|
||||
const result = await this.prometheusClient.query(
|
||||
|
||||
const usage = new Map<string, ReportedUsage>();
|
||||
const processResults = (
|
||||
data: QueryResultData,
|
||||
setValue: (entry: ReportedUsage, value: string) => void
|
||||
) => {
|
||||
for (const result of data.result) {
|
||||
const country = result.metric['location'] || '';
|
||||
const asn = result.metric['asn'] ? Number(result.metric['asn']) : undefined;
|
||||
const key = `${country}-${asn}`;
|
||||
const entry = usage.get(key) || {
|
||||
country,
|
||||
asn,
|
||||
inboundBytes: 0,
|
||||
tunnelTimeSec: 0,
|
||||
};
|
||||
setValue(entry, result.value[1]);
|
||||
if (!usage.has(key)) {
|
||||
usage.set(key, entry);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Query and process inbound data bytes by country+ASN.
|
||||
const dataBytesQueryResponse = await this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|p<t"}[${timeDeltaSecs}s])) by (location, asn)`
|
||||
);
|
||||
const usage = [] as ReportedUsage[];
|
||||
for (const entry of result.result) {
|
||||
const country = entry.metric['location'] || '';
|
||||
const asn = entry.metric['asn'] ? Number(entry.metric['asn']) : undefined;
|
||||
const inboundBytes = Math.round(parseFloat(entry.value[1]));
|
||||
usage.push({country, inboundBytes, asn});
|
||||
}
|
||||
return usage;
|
||||
processResults(dataBytesQueryResponse, (entry, value) => {
|
||||
entry.inboundBytes = Math.round(parseFloat(value));
|
||||
});
|
||||
|
||||
// Query and process tunneltime by country+ASN.
|
||||
const tunnelTimeQueryResponse = await this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_tunnel_time_seconds_per_location[${timeDeltaSecs}s])) by (location, asn)`
|
||||
);
|
||||
processResults(tunnelTimeQueryResponse, (entry, value) => {
|
||||
entry.tunnelTimeSec = Math.round(parseFloat(value));
|
||||
});
|
||||
|
||||
return Array.from(usage.values());
|
||||
}
|
||||
|
||||
reset() {
|
||||
|
|
@ -205,7 +235,7 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher {
|
|||
|
||||
const userReports: HourlyUserMetricsReportJson[] = [];
|
||||
for (const locationUsage of locationUsageMetrics) {
|
||||
if (locationUsage.inboundBytes === 0) {
|
||||
if (locationUsage.inboundBytes === 0 && locationUsage.tunnelTimeSec === 0) {
|
||||
continue;
|
||||
}
|
||||
if (isSanctionedCountry(locationUsage.country)) {
|
||||
|
|
@ -215,8 +245,9 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher {
|
|||
// It's used to differentiate the row from the legacy key usage rows.
|
||||
const country = locationUsage.country || 'ZZ';
|
||||
const report: HourlyUserMetricsReportJson = {
|
||||
bytesTransferred: locationUsage.inboundBytes,
|
||||
countries: [country],
|
||||
bytesTransferred: locationUsage.inboundBytes,
|
||||
tunnelTimeSec: locationUsage.tunnelTimeSec,
|
||||
};
|
||||
if (locationUsage.asn) {
|
||||
report.asn = locationUsage.asn;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue