mirror of
https://github.com/OutlineFoundation/outline-server.git
synced 2026-08-04 14:37:34 +00:00
feat(server): unified metrics endpoint (#1616)
* draft of unified metrics endpoint * ok, empty state works * I think it's there? * prefix with 'fake' * some feedback * continue feedback * test refactor * remove unused imports * bad refactor * partial feedback * move time range parsing logic * full union * feedback
This commit is contained in:
Daniel LaCosse
GitHub
parent
f558117b29
commit
9d9f76b9a5
8 changed files with 433 additions and 10 deletions
|
|
@ -37,7 +37,11 @@ interface QueryResult {
|
|||
error: string;
|
||||
}
|
||||
|
||||
export class PrometheusClient {
|
||||
export interface PrometheusClient {
|
||||
query(query: string): Promise<QueryResultData>;
|
||||
}
|
||||
|
||||
export class ApiPrometheusClient implements PrometheusClient {
|
||||
constructor(private address: string) {}
|
||||
|
||||
query(query: string): Promise<QueryResultData> {
|
||||
|
|
@ -101,7 +105,7 @@ async function spawnPrometheusSubprocess(
|
|||
prometheusEndpoint: string
|
||||
): Promise<child_process.ChildProcess> {
|
||||
logging.info('======== Starting Prometheus ========');
|
||||
logging.info(`${binaryFilename} ${processArgs.map(a => `"${a}"`).join(' ')}`);
|
||||
logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`);
|
||||
const runProcess = child_process.spawn(binaryFilename, processArgs);
|
||||
runProcess.on('error', (error) => {
|
||||
logging.error(`Error spawning Prometheus: ${error}`);
|
||||
|
|
|
|||
|
|
@ -127,7 +127,60 @@ paths:
|
|||
responses:
|
||||
'204':
|
||||
description: Access key limit deleted successfully.
|
||||
|
||||
/experimental/server/metrics:
|
||||
get:
|
||||
tags: Server
|
||||
parameters:
|
||||
- in: query
|
||||
name: since
|
||||
description: the range of time to return data for
|
||||
schema:
|
||||
type: string
|
||||
responses:
|
||||
'200':
|
||||
description: Display server metric information
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
server:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
location:
|
||||
type: string
|
||||
asn:
|
||||
type: number
|
||||
asOrg:
|
||||
type: string
|
||||
tunnelTime:
|
||||
type: object
|
||||
properties:
|
||||
seconds: number
|
||||
dataTransferred:
|
||||
type: object
|
||||
properties:
|
||||
bytes: number
|
||||
accessKeys:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
accessKeyId:
|
||||
type: string
|
||||
tunnelTime:
|
||||
type: object
|
||||
properties:
|
||||
seconds: number
|
||||
dataTransferred:
|
||||
type: object
|
||||
properties:
|
||||
bytes: number
|
||||
examples:
|
||||
'0':
|
||||
value: '{"server":[{"location":"US","asn":null,"asOrg":null,"tunnelTime":{"seconds":100},"dataTransferred":{"bytes":100}}],"accessKeys":[{"accessKeyId":"0","tunnelTime":{"seconds":100},"dataTransferred":{"bytes":100}}]}'
|
||||
/name:
|
||||
put:
|
||||
description: Renames the server
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ import {RealClock} from '../infrastructure/clock';
|
|||
import {PortProvider} from '../infrastructure/get_port';
|
||||
import * as json_config from '../infrastructure/json_config';
|
||||
import * as logging from '../infrastructure/logging';
|
||||
import {PrometheusClient, startPrometheus} from '../infrastructure/prometheus_scraper';
|
||||
import {ApiPrometheusClient, startPrometheus} from '../infrastructure/prometheus_scraper';
|
||||
import {RolloutTracker} from '../infrastructure/rollout';
|
||||
import * as version from './version';
|
||||
|
||||
|
|
@ -197,7 +197,7 @@ async function main() {
|
|||
prometheusEndpoint
|
||||
);
|
||||
|
||||
const prometheusClient = new PrometheusClient(prometheusEndpoint);
|
||||
const prometheusClient = new ApiPrometheusClient(prometheusEndpoint);
|
||||
if (!serverConfig.data().portForNewAccessKeys) {
|
||||
serverConfig.data().portForNewAccessKeys = await portProvider.reserveNewPort();
|
||||
serverConfig.write();
|
||||
|
|
|
|||
|
|
@ -13,9 +13,200 @@
|
|||
// limitations under the License.
|
||||
|
||||
import {PrometheusManagerMetrics} from './manager_metrics';
|
||||
import {PrometheusClient, QueryResultData} from '../infrastructure/prometheus_scraper';
|
||||
import {FakePrometheusClient} from './mocks/mocks';
|
||||
|
||||
export class QueryMapPrometheusClient implements PrometheusClient {
|
||||
constructor(private queryMap: {[query: string]: QueryResultData}) {}
|
||||
|
||||
async query(_query: string): Promise<QueryResultData> {
|
||||
return this.queryMap[_query];
|
||||
}
|
||||
}
|
||||
|
||||
describe('PrometheusManagerMetrics', () => {
|
||||
it('getServerMetrics', async (done) => {
|
||||
const managerMetrics = new PrometheusManagerMetrics(
|
||||
new QueryMapPrometheusClient({
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[0s])) by (location, asn, asorg)':
|
||||
{
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: null,
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0s])) by (location, asn, asorg)':
|
||||
{
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: null,
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[0s])) by (access_key)': {
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
access_key: '0',
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_tunnel_time_seconds[0s])) by (access_key)': {
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
access_key: '0',
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const serverMetrics = await managerMetrics.getServerMetrics({seconds: 0});
|
||||
|
||||
expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{
|
||||
"server": [
|
||||
{
|
||||
"location": "US",
|
||||
"asn": 49490,
|
||||
"asOrg": "null",
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
},
|
||||
"dataTransferred": {
|
||||
"bytes": 1000
|
||||
}
|
||||
}
|
||||
],
|
||||
"accessKeys": [
|
||||
{
|
||||
"accessKeyId": 0,
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
},
|
||||
"dataTransferred": {
|
||||
"bytes": 1000
|
||||
}
|
||||
}
|
||||
]
|
||||
}`);
|
||||
done();
|
||||
});
|
||||
|
||||
it('getServerMetrics - does a full outer join on metric data', async (done) => {
|
||||
const managerMetrics = new PrometheusManagerMetrics(
|
||||
new QueryMapPrometheusClient({
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[0s])) by (location, asn, asorg)':
|
||||
{
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: null,
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_tunnel_time_seconds_per_location[0s])) by (location, asn, asorg)':
|
||||
{
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'CA',
|
||||
asn: '53520',
|
||||
asorg: null,
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[0s])) by (access_key)': {
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
access_key: '0',
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_tunnel_time_seconds[0s])) by (access_key)': {
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
access_key: '1',
|
||||
},
|
||||
value: [null, '1000'],
|
||||
},
|
||||
],
|
||||
},
|
||||
})
|
||||
);
|
||||
|
||||
const serverMetrics = await managerMetrics.getServerMetrics({seconds: 0});
|
||||
|
||||
expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{
|
||||
"server": [
|
||||
{
|
||||
"location": "CA",
|
||||
"asn": 53520,
|
||||
"asOrg": "null",
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
}
|
||||
},
|
||||
{
|
||||
"location": "US",
|
||||
"asn": 49490,
|
||||
"asOrg": "null",
|
||||
"dataTransferred": {
|
||||
"bytes": 1000
|
||||
}
|
||||
}
|
||||
],
|
||||
"accessKeys": [
|
||||
{
|
||||
"accessKeyId": 1,
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
}
|
||||
},
|
||||
{
|
||||
"accessKeyId": 0,
|
||||
"dataTransferred": {
|
||||
"bytes": 1000
|
||||
}
|
||||
}
|
||||
]
|
||||
}`);
|
||||
done();
|
||||
});
|
||||
|
||||
it('getOutboundByteTransfer', async (done) => {
|
||||
const managerMetrics = new PrometheusManagerMetrics(
|
||||
new FakePrometheusClient({'access-key-1': 1000, 'access-key-2': 10000})
|
||||
|
|
|
|||
|
|
@ -15,8 +15,36 @@
|
|||
import {PrometheusClient} from '../infrastructure/prometheus_scraper';
|
||||
import {DataUsageByUser, DataUsageTimeframe} from '../model/metrics';
|
||||
|
||||
interface Duration {
|
||||
seconds: number;
|
||||
}
|
||||
|
||||
interface Data {
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
interface ServerMetricsServerEntry {
|
||||
location: string;
|
||||
asn: number;
|
||||
asOrg: string;
|
||||
tunnelTime: Duration;
|
||||
dataTransferred: Data;
|
||||
}
|
||||
|
||||
interface ServerMetricsAccessKeyEntry {
|
||||
accessKeyId: number;
|
||||
tunnelTime: Duration;
|
||||
dataTransferred: Data;
|
||||
}
|
||||
|
||||
interface ServerMetrics {
|
||||
server: ServerMetricsServerEntry[];
|
||||
accessKeys: ServerMetricsAccessKeyEntry[];
|
||||
}
|
||||
|
||||
export interface ManagerMetrics {
|
||||
getOutboundByteTransfer(timeframe: DataUsageTimeframe): Promise<DataUsageByUser>;
|
||||
getServerMetrics(timeframe: Duration): Promise<ServerMetrics>;
|
||||
}
|
||||
|
||||
// Reads manager metrics from a Prometheus instance.
|
||||
|
|
@ -40,4 +68,83 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
}
|
||||
return {bytesTransferredByUserId: usage};
|
||||
}
|
||||
|
||||
async getServerMetrics(timeframe: Duration): Promise<ServerMetrics> {
|
||||
const dataTransferredByLocation = await this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[${timeframe.seconds}s])) by (location, asn, asorg)`
|
||||
);
|
||||
const tunnelTimeByLocation = await this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_tunnel_time_seconds_per_location[${timeframe.seconds}s])) by (location, asn, asorg)`
|
||||
);
|
||||
const dataTransferredByAccessKey = await this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[${timeframe.seconds}s])) by (access_key)`
|
||||
);
|
||||
const tunnelTimeByAccessKey = await this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_tunnel_time_seconds[${timeframe.seconds}s])) by (access_key)`
|
||||
);
|
||||
|
||||
const serverMap = new Map();
|
||||
const serverMapKey = (entry) =>
|
||||
`${entry.metric['location']},${entry.metric['asn']},${entry.metric['asorg']}`;
|
||||
for (const entry of tunnelTimeByLocation.result) {
|
||||
serverMap.set(serverMapKey(entry), {
|
||||
tunnelTime: {
|
||||
seconds: parseFloat(entry.value[1]),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
for (const entry of dataTransferredByLocation.result) {
|
||||
if (!serverMap.has(serverMapKey(entry))) {
|
||||
serverMap.set(serverMapKey(entry), {});
|
||||
}
|
||||
|
||||
serverMap.get(serverMapKey(entry)).dataTransferred = {
|
||||
bytes: parseFloat(entry.value[1]),
|
||||
};
|
||||
}
|
||||
|
||||
const server = [];
|
||||
for (const [key, metrics] of serverMap.entries()) {
|
||||
const [location, asn, asOrg] = key.split(',');
|
||||
server.push({
|
||||
location,
|
||||
asn: parseInt(asn),
|
||||
asOrg,
|
||||
...metrics,
|
||||
});
|
||||
}
|
||||
|
||||
const accessKeyMap = new Map();
|
||||
for (const entry of tunnelTimeByAccessKey.result) {
|
||||
accessKeyMap.set(entry.metric['access_key'], {
|
||||
tunnelTime: {
|
||||
seconds: parseFloat(entry.value[1]),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
for (const entry of dataTransferredByAccessKey.result) {
|
||||
if (!accessKeyMap.has(entry.metric['access_key'])) {
|
||||
accessKeyMap.set(entry.metric['access_key'], {});
|
||||
}
|
||||
|
||||
accessKeyMap.get(entry.metric['access_key']).dataTransferred = {
|
||||
bytes: parseFloat(entry.value[1]),
|
||||
};
|
||||
}
|
||||
|
||||
const accessKeys = [];
|
||||
for (const [key, metrics] of accessKeyMap.entries()) {
|
||||
accessKeys.push({
|
||||
accessKeyId: parseInt(key),
|
||||
...metrics,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
server,
|
||||
accessKeys,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import * as restify from 'restify';
|
|||
import {InMemoryConfig, JsonConfig} from '../infrastructure/json_config';
|
||||
import {AccessKey, AccessKeyRepository, DataLimit} from '../model/access_key';
|
||||
import {ManagerMetrics} from './manager_metrics';
|
||||
import {bindService, ShadowsocksManagerService} from './manager_service';
|
||||
import {bindService, ShadowsocksManagerService, convertTimeRangeToSeconds} from './manager_service';
|
||||
import {FakePrometheusClient, FakeShadowsocksServer} from './mocks/mocks';
|
||||
import {AccessKeyConfigJson, ServerAccessKeyRepository} from './server_access_key';
|
||||
import {ServerConfigJson} from './server_config';
|
||||
|
|
@ -1200,6 +1200,20 @@ describe('bindService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe('convertTimeRangeToHours', () => {
|
||||
it('properly parses time ranges', () => {
|
||||
expect(convertTimeRangeToSeconds('30d')).toEqual(30 * 24 * 60 * 60);
|
||||
expect(convertTimeRangeToSeconds('20h')).toEqual(20 * 60 * 60);
|
||||
expect(convertTimeRangeToSeconds('3w')).toEqual(7 * 3 * 24 * 60 * 60);
|
||||
});
|
||||
|
||||
it('throws when an invalid time range is provided', () => {
|
||||
expect(() => convertTimeRangeToSeconds('30dd')).toThrow();
|
||||
expect(() => convertTimeRangeToSeconds('hi mom')).toThrow();
|
||||
expect(() => convertTimeRangeToSeconds('1j')).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
class ShadowsocksManagerServiceBuilder {
|
||||
private defaultServerName_ = 'default name';
|
||||
private serverConfig_: JsonConfig<ServerConfigJson> = null;
|
||||
|
|
|
|||
|
|
@ -75,10 +75,19 @@ interface RequestParams {
|
|||
// method: string
|
||||
[param: string]: unknown;
|
||||
}
|
||||
|
||||
// Type to reflect that we recive an untyped query string
|
||||
interface RequestQuery {
|
||||
// Supported parameters:
|
||||
// since: string
|
||||
[param: string]: unknown;
|
||||
}
|
||||
|
||||
// Simplified request and response type interfaces containing only the
|
||||
// properties we actually use, to make testing easier.
|
||||
interface RequestType {
|
||||
params: RequestParams;
|
||||
query?: RequestQuery;
|
||||
}
|
||||
interface ResponseType {
|
||||
send(code: number, data?: {}): void;
|
||||
|
|
@ -122,6 +131,7 @@ export function bindService(
|
|||
|
||||
apiServer.put(`${apiPrefix}/name`, service.renameServer.bind(service));
|
||||
apiServer.get(`${apiPrefix}/server`, service.getServer.bind(service));
|
||||
apiServer.get(`${apiPrefix}/experimental/server/metrics`, service.getServerMetrics.bind(service));
|
||||
apiServer.put(
|
||||
`${apiPrefix}/server/access-key-data-limit`,
|
||||
service.setDefaultDataLimit.bind(service)
|
||||
|
|
@ -180,6 +190,24 @@ function redirect(url: string): restify.RequestHandlerType {
|
|||
};
|
||||
}
|
||||
|
||||
export function convertTimeRangeToSeconds(timeRange: string): number {
|
||||
const TIME_RANGE_UNIT_TO_SECONDS_MULTIPLYER = {
|
||||
s: 1,
|
||||
h: 60 * 60,
|
||||
d: 24 * 60 * 60,
|
||||
w: 7 * 24 * 60 * 60,
|
||||
};
|
||||
|
||||
const timeRangeValue = Number(timeRange.slice(0, -1));
|
||||
const timeRangeUnit = timeRange.slice(-1);
|
||||
|
||||
if (isNaN(timeRangeValue) || !TIME_RANGE_UNIT_TO_SECONDS_MULTIPLYER[timeRangeUnit]) {
|
||||
throw new TypeError(`Invalid time range: ${timeRange}`);
|
||||
}
|
||||
|
||||
return timeRangeValue * TIME_RANGE_UNIT_TO_SECONDS_MULTIPLYER[timeRangeUnit];
|
||||
}
|
||||
|
||||
function validateAccessKeyId(accessKeyId: unknown): string {
|
||||
if (!accessKeyId) {
|
||||
throw new restifyErrors.MissingParameterError({statusCode: 400}, 'Parameter `id` is missing');
|
||||
|
|
@ -599,6 +627,34 @@ export class ShadowsocksManagerService {
|
|||
}
|
||||
}
|
||||
|
||||
async getServerMetrics(req: RequestType, res: ResponseType, next: restify.Next) {
|
||||
logging.debug(`getServerMetrics request ${JSON.stringify(req.params)}`);
|
||||
|
||||
let seconds;
|
||||
try {
|
||||
if (!req.query?.since) {
|
||||
return next(
|
||||
new restifyErrors.MissingParameterError({statusCode: 400}, 'Parameter `since` is missing')
|
||||
);
|
||||
}
|
||||
|
||||
seconds = convertTimeRangeToSeconds(req.query.since as string);
|
||||
} catch (error) {
|
||||
logging.error(error);
|
||||
return next(new restifyErrors.InvalidArgumentError({statusCode: 400}, error.message));
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await this.managerMetrics.getServerMetrics({seconds});
|
||||
res.send(HttpSuccess.OK, response);
|
||||
logging.debug(`getServerMetrics response ${JSON.stringify(response)}`);
|
||||
return next();
|
||||
} catch (error) {
|
||||
logging.error(error);
|
||||
return next(new restifyErrors.InternalServerError());
|
||||
}
|
||||
}
|
||||
|
||||
getShareMetrics(req: RequestType, res: ResponseType, next: restify.Next): void {
|
||||
logging.debug(`getShareMetrics request ${JSON.stringify(req.params)}`);
|
||||
const response = {metricsEnabled: this.metricsPublisher.isSharingEnabled()};
|
||||
|
|
|
|||
|
|
@ -48,10 +48,8 @@ export class FakeShadowsocksServer implements ShadowsocksServer {
|
|||
}
|
||||
}
|
||||
|
||||
export class FakePrometheusClient extends PrometheusClient {
|
||||
constructor(public bytesTransferredById: {[accessKeyId: string]: number}) {
|
||||
super('');
|
||||
}
|
||||
export class FakePrometheusClient implements PrometheusClient {
|
||||
constructor(public bytesTransferredById: {[accessKeyId: string]: number}) {}
|
||||
|
||||
async query(_query: string): Promise<QueryResultData> {
|
||||
const queryResultData = {result: []} as QueryResultData;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue