Rename stats to metrics

This commit is contained in:
fortuna 2018-09-11 15:04:04 -04:00
parent 9d19cf97f9
commit a7641eefb2
15 changed files with 209 additions and 96 deletions

52
coreos.txt Normal file
View file

@ -0,0 +1,52 @@
-----BEGIN RSA PRIVATE KEY-----
MIIJKQIBAAKCAgEAy8RtwobMKZCcvVBkP6oQqb0moUJEv1N2G166p/fr0DjoK8Ei
70xjKGcqDFhXvR75oW4DjSL1jUwSXfI4L8b3Kd71UBqJD2Xpb6upLII6MxIxnanU
hI0a0R7x8G64GgC204YmMB7HN2Z8rxGrC7H0qDCUbn+vKEBSfSxp99XRgRBzk1Z0
sNPGd0xxDh8Gsm7ViGuDmyEv5ICCrAMVr2vEjyHczqb9FQe7/7JSKrY3rrOFcH0k
V0wtw9nT5wp9uJGxO0EYy5AYma7CKeUC46ijLB16rbNc22FaezsMTHbRnXpks+KT
0Pt8DLO5tjQsZ4yo5vZDb9WzWuMdUkkBIHw/9gUP3Hb+OKjdsdjc6A+sfKXrotBA
UdirlMTs2FzRpDJhAg9qTL/mngJtEBN7wXLhXKPXCZdtH/dBX91QZBbIxNaqeY/A
AKG6VZONGgZcVtU1ix8+uqijABkWprhVLO0jBxkSPZiMy2iTz6p+FJ+zy/ZbRfl+
YZGjhs6xdklhll1zrCSXBcY2aAUsgQWwZl8hG1ibTcFGq0lxCRdcXeYwX8uuRFwE
ZFBq3yG6SZOR4JZLWrEgLHdb6T7rik2U73cyX+/3x+j2QG65mvqLdisaja3d/961
Tjj4qFliz5ZWPECVWNigqnmqSe/nv5XPC2A93GM25PHFY+bJlpaaq0QWP1ECAwEA
AQKCAgBZgFP9p5uxfhV6if8ef6KGC9EV77emmhA8gWVXNexcL0K7RUAE//Zl3rp3
Za5UIXDgWSQyL/LPN2Sx4xyOz8PsnkP+BUnCe68HH81VAXZyzEEC0X/JIPlwdTkr
tFYlBb9INZo8dKhoSxnlA8uvfWDLJ1trFaZn9ajF1mZNN5uoJwO86bKjoMGB0Q+v
di1I3qnoG+FYmEEjCtdamphBzwItJGCKXIq5XAZVj4vLuvHGSJAKEs2NkqZfaiRL
TS5fjY7dSgCMGSTVDA+4uyCDwqS5UdF6zlew/JfznMIQK/hyRTpKUPFAT9Xy9lZS
E6SVbxEZMX35d1IqT6unYu2dyTWGHolUNieFnUCZsXCyOl7lqY0OGSEMAsd+kFDD
LMDkq/7QffJhQve2Av5kwKuPeUt/wzwJ+RPVawxHOPZNwcA7qeGzTcClgUu+1LMo
jxuLbV9pcMfv07fD2Rf5Cm2EW3lY5aj0/D45sia6IjMYrwUzFHztmUcvO/hj6Kur
Jmbyuw1Dn44StXWYrSef0aWVqmKkWh8NY03mYuLFOEhkvaaAj+dGjX3gIcqAbC+f
GjCXb7vWGL/RanNVmaZVu3W9zTURHZXbeKU7mUZ7y5gug1mIcfaONKsQ9I/eiZaP
uZmFfrePjzvIlBvG5mVhdm8U9DC1PlEz9VxYDFy/j08Nkx0F4QKCAQEA/Tm90Gr6
e63kZgX+KraFdnCnoDJF2WaiDRAmDikc4tFwaAcahp84Z6Z/OkYw3fq4786rsusf
QBqACg4mKl0V//atGNBCPc0rResHkOpqW7+mwpxT28hvL6zrFRJiRqbG5MSIkL/A
2i/f6po7g8voye4gHtdRHPKdERJ8Juzdm2ZdTLSo0XOT6rlvwECHHvEXlHfugHDd
zNEqJfpwU6E/nU4o6FTOO3xZw2SOnCNdR/Am3xdJFHRFIrZ1/0sGv5EU+HUtDfEq
9jmmWaZYF2pjCIT82EQ00tNpBvwQZ54aQk/xukxbCGR+nycJ9cMl3RbzumnZTtcj
WiMw+rVOUFrWJQKCAQEAzf/265/bAqPC+NbGOR28WBAI2Ja+QRVgEiJaSVt7ngDm
VAhVF0fHjFRX+c8YjR6fKUPHzQOAV80xWN/k9JEg63b58whm/QVV+5WcQ/9yD2SB
6YtBimzT40c+b39Y7tNLVGQ8n512jVHGKBuHjqWmXVI37FxkbtxHBWSaiCz2EVyY
BvMmsaHjCYgr2cNnOmwJjsjBLqDR1mZrds1LqYu+kM6THLTauPhj+i3cCH3ZH2gU
0EG//S14WMnZuBrVv6BPzIeO09PwFEE7KzPSSwoB9H0HH3lIyQxoYLWr2OOBDz9s
kmQ/piqSbtO+ARWYd86oWjLjWF4G/HvjVzeU9cWuvQKCAQEAsDIaOkgFrEMt9jNb
TBseOHBgop34bjH0tgQzhozi1YwHm8q9kUh+ddirFEA8xmgrgGkcnWzunKsTxmtb
8QQ+R5E7llVqkhgNcSP9ar9BbD+paCZgT0Bi5Rh7pnjZOvHW2N1LbPSP2wGO222f
1a/vdXokjXEitnK2CWgETQ1pkTSj3Lij8sFp/dwzvuDnZAc7cgoVQPfHzTkJC837
lKVRX0JAQpCnw0peJw/0Dv6obGLUmUxZhEr2xBWTeySYOHlZzxuxUs8pJpSshBqz
lu9mo0ntqQmke8GwhbSkMYUYHmYD+64fdXJ/jHwceQ3lbbYHtwDpvOsDZTexX/EB
4fWipQKCAQAVT7G/1p3VvBNjovSG3CisA5ymq5GrMgbqWVt1010Kj5VEhEgpTGe8
gM1JLr+fedeFcVmuP/p7GuNMCn2pP4pkUb6yAeCFtJOcn3G3JyoppYA7JQj2xSN2
k9xFtKsCqiFU7bnH2YZ2QEt7wr1XaJO5e9QFQ9mwDmHakPnbtKsQSMABmA4cul6+
kbPXp3t6c8rZVrOFm2WaBKaBd+On+qkQWg6mHZ+zGx9ctnnY9wwLT703flXaX2Xf
6aH4he4vEOqwgiWojHh93/G7GnVbBgIFxRmDjAyXoAz8VE8e9QpZBXq5+l0LV8Qm
awlxG0bWvi50hmc2sSOP41E1qK5kbrsRAoIBAQDUDpgJ2HrCKnkIFj3IzPHnw/Ur
+bmWUtfCCP07THH7dYMSfZLMwXwwJeNHtJIo4TUqghcTSH+QJYhCwOkwwexa/F4Z
E3FRy/KBlsjIcL/bEelCK8PpsZd9fYZkrHK+hz7uyiXULrErbkiQo0akze1DghCw
R0gSgkRj2YLZ+2e5Tud/eDC1l9JbVWuAOj1n2/MdxsRIMSMBdTRaivLHuWVi83Gx
yyGuJxy/H9Z0izmDJzbPl3ETf6OLLvJY3tL8qgQrQWajvu8dvGyi2F4KpuaLu29b
pTg7GLNkzLcltLZpcJf9sk0+XJP1Ky07Cqg0qfVSBIzyCQ5XQgAnXy67lOWK
-----END RSA PRIVATE KEY-----

51
key.txt Normal file
View file

@ -0,0 +1,51 @@
-----BEGIN RSA PRIVATE KEY-----
MIIJKAIBAAKCAgEAnrDtsh/DFaawDjY1XOijPpk1dpJlcKibSLfrTeGJAepT1IS2
h8s/V1PP4WOqCpHAoukzcJFbyIhAFUsuXlpY32YEAKIHA0dEHvdq/GgluylyZqLv
2r2II+Fm01142ZXQGg+SEI4i0+fkZsWU7HnbmBydJKFpyn5yQv3LMwhaPCZmg5B4
SxnkeigfrmxfKP2xZ9qoxNivVulf0mlcRl/UkjXqFUCAR6zCUqiMIdZ/JYiaMd+3
39owVqiTBo2+AdtiO4GxQ964NwqcfL7cdhHslKt8/MUsOuTkAuMWT41SiwUih6nk
E87rgwQbg4Iv70YEcqR6d0JR8VWj8pfPnSTECN4XYFdkE59udS5q6R95v2VF2EKL
MV4R5S+DpXraVtg5qxkTBXXQACfoKOr/PDCG3SbAJylORHML7JMPpN4Ovt2cft8x
Fkrrv9bCL8duMHSrVd+jlcYb7KWB8kDMSIkI6j6Vgj+FsuHGwTzAHdwikrCNvX2z
SH+hhz8QuzOb7HyTCFRBeueb3AMm5a3UAn1BHdsY+sU/8hqF3VUHx6Eh0A1wAShv
kU2S8w4XkFsiam0fJ9pPes/UIvYCsOpkq/Cyhxxv1DW+3iqDqT3cbvJArNN16iyT
CZMeejkXUP61dQKevc8wbtHHTsN1HZAagIJZGOTiJmErHv8X4w5U5SIRBEUCAwEA
AQKCAgAMUy2C4hiiu9l8oRUv7BmRqRCP7vrV7yJwWWh7GgMsFOkB3nTWwyBeRfIl
TTpNq9hMTtRh78gzIPHpNXeaeGXio+e3rN3ikUxnI0w54lTb3nI2Kn16fbHvJ3h2
/hF+xLXu8Dql8oQd9Sq2GK5iV2yIueAykh7HTV7OeSupAQMRHOJlYXkWTCKEok0j
nOMfKeT3bfIhp7qmg2Wfz/MMvDCkUm+lyuarqm1FQjXwAtrJLXzcVsXPKwEKGfmh
TqztM/7DJGWCIfAoxjg1MBWrTeUT/uWsNBwlTtWuq7h14UIB7hoqczV1nGKKQG1q
n1fdu5Bo3wFr9cird71OACBt0gcvkctVejgwTNg9z/M9dLFvTguty2Hwbb1wCg8F
VxKXhDxUxpcl+1MQKqPSxWcSv7v6Zhtdj5sEwPF3Q2MUV51geKP1Ds/0OrOr88Lb
J1TaD53lFe5TIHM70CVswy+Nn64y7f/7zDi5OPabmYEkyUcu0O3aPIEVHxf3FwZD
/G5nMs4sRoaFGuMiOIbXg0Gw6MJ9eQTdWxFSK27P0gncsDZrtu4pi87i3Rt4917T
5pIrduUsqIILbpyo17pvpufkXPrukgEHjbWejl3s1wleWVktWb7D45mppmXaSuc7
aM++O43W97DWnmSCbhoD/y8dF8bJ3f65ekkzNeRUuUU2zdhxgQKCAQEA0W4Qg/Z5
hnBROWsZNddiiEWUAASYgMGdtPKj7CYovVhL+1zX0wHp371k0TyOwbTh+Pg+S5z9
rP+9/yltMyx5dToVF20zsCmfVPzGl1ksIiiaJfy70IPTVT2bj5sFgv+bnTRibXgN
LDyJpCUR4Wfzl8VBgqUYSyZc6WVH35SKvudfMgvOKtHkb56Pk5gRI3O5VyDUHL4g
8k1HCOoVxR1w5vZkSmImQaI2jLZ0vl6F1w1SlADci/jyNRV/3y7frAYH73v2pKJ3
iTgF8+cZXsgHP9UkSHA0CXM/h62DVh1iWRXEyzWdGe9roivmPbAo//mGtYLhHGge
kT1oycZ8DPjDcQKCAQEAwfqGKqfq62YEKUC44YrTLrsRtepKa8FAkD2MEHeyPOMh
nA5ECyoNJKjViqEsb8og5USOfOV223ZLGis8Qnx4Z9wkO/7g9ky0zeBsZbIrBpFi
T1vCLmi1Qh9GqvCkiIJ8lvkGQq/tCvLnBLOtM/dFqs4oxns++5fsuWLf7gL5eDHk
pb2W8HrZHWMdxHiwv+5GD8WJI35vcmgaVBQaKrkIrRHhkklvrJCRtmZ5y1WlKUzm
ywy8NSphuJ0nUdjKHKy4dufq6HbRQWjZQczT4VnujuT9syvsxVfoEjRGKZeeNw+I
+4o+fnNbZ3mYOtJWW6cB5oQDIWu3JSZ+Anjhb0K8FQKCAQEAzX4yxFkm+uvgNvfI
P9U8ADxNMQtRXB0eknr2rvLuTIOD4ntB5fBdu8TJVKkX5ieHBtUFwwmiu4ogsmrC
lFDSSF0abucerX7ZsPlqHv1HWaj/P8DRxJJk3aHarrjMWrJVzZWl8oW2Xy5zW9Vn
ywVFtii90+QMh0h4KCbRtCa7UQATnzaIL+nNPFyXwpmWT3PwavZySlWgXD/JMI0H
mWb+7hDbbUULBqGU5tLskBKNPur6mPCTduBpP/79fk8u90rfpHO9GeO0aLbI2H5s
nVymCFMqC83UsWUc9BMj6G5insjGVSIhKV6L/Q8YFnVwdWIwdI+cNFRRke9wj3Or
Kss4cQKCAQAEvFkKMY9Kr/LqDup0ly8QtQB5sH6gotcwrk/9Fu8DDYiEhtSicSRh
AL415DlxgT3MWyAfbHq6YOj0epm+BcvqvTUlQdO8L6M6Y4BB+1eRkXsU9OiIuYWz
V5AiHD3oF0dzaCD+/8yJt+Rr+PcBjcflo6LbNacT/WGKJR/Sb8AnbxBl+3rz5Avo
68KOSWQHS4nqWKhAdZXC9UevRc5dvKa5kvYu3Bwd2mm0Skwu6qhdmcMIsgbmRWKd
XzjWhrRofs8CGCdkBYKWVjj3okiJ9+gbFPwco7XkG4FO8HfGDC2QqpBtk7Jy494X
aKCOzxPMqQci2ZY5+qc+APKSnODkFn0BAoIBAEeov1NmKjF9YLzDnCYKpEzT0/CS
K8Ipcq3B5M1wOanrv9lFdSZdmdqY5XWTcSgjNfE75kALUstiUoUVBxmIZY8b/I2t
Vun3shvoHv1Bxj8qKnOPBG93lUC6Xs3/9HrvmXJGPk+keMiglFLPNFmZWV9ElkIv
6Q1E2kE3oGFUmhfQAqJ302QrHnIvcocfrxtsN3rh7s0NTrsadEJunzeW1C8SYXLO
gZtvv5Iw4ulgQD7f3Y7VbUlqarWztLiuGujjlts/jkqMCp+dgWEMPjGxxnyVHk3a
0MQPWJrhqHxBq25tAxu3Gd33WDOPL9wceyyVnpvIUVj7zU35h/sJUM7aHrg=
-----END RSA PRIVATE KEY-----

View file

@ -47,7 +47,7 @@ export class FileConfig<T> implements JsonConfig<T> {
write() {
// Write to temporary file, then move that temporary file to the
// persistent location, to avoid accidentally breaking the stats file.
// persistent location, to avoid accidentally breaking the metrics file.
// Use *Sync calls for atomic operations, to guard against corrupting
// these files.
const tempFilename = `${this.filename}.${Date.now()}`;

View file

@ -15,15 +15,15 @@
import {AccessKeyId} from './access_key';
export type LastHourMetricsReadyCallback =
(startDatetime: Date, endDatetime: Date, lastHourUserStats: Map<AccessKeyId, PerUserStats>) =>
void;
(startDatetime: Date, endDatetime: Date,
lastHourUserMetrics: Map<AccessKeyId, PerUserMetrics>) => void;
export interface PerUserStats {
export interface PerUserMetrics {
bytesTransferred: number;
anonymizedIpAddresses: Set<string>;
}
// Byte transfer stats for the past 30 days, including both inbound and outbound.
// Byte transfer metrics for the past 30 days, including both inbound and outbound.
// TODO: this is copied at src/model/server.ts. Both copies should
// be kept in sync, until we can find a way to share code between the web_app
// and shadowbox.

View file

@ -16,7 +16,7 @@ import * as dgram from 'dgram';
export interface ShadowsocksServer {
startInstance(
portNumber: number, password: string, statsSocket: dgram.Socket,
portNumber: number, password: string, metricsSocket: dgram.Socket,
encryptionMethod?: string): Promise<ShadowsocksInstance>;
}

View file

@ -30,17 +30,17 @@ export class LibevShadowsocksServer implements ShadowsocksServer {
constructor(private publicAddress: string, private verbose: boolean) {}
public startInstance(
portNumber: number, password: string, statsSocket: dgram.Socket,
portNumber: number, password: string, metricsSocket: dgram.Socket,
encryptionMethod = this.DEFAULT_METHOD): Promise<ShadowsocksInstance> {
logging.info(`Starting server on port ${portNumber}`);
const statsAddress = statsSocket.address();
const metricsAddress = metricsSocket.address();
const commandArguments = [
'-m', encryptionMethod, // Encryption method
'-u', // Allow UDP
'--fast-open', // Allow TCP fast open
'-p', portNumber.toString(), '-k', password, '--manager-address',
`${statsAddress.address}:${statsAddress.port}`
`${metricsAddress.address}:${metricsAddress.port}`
];
logging.info('starting ss-server with args: ' + commandArguments.join(' '));
// Add the system DNS servers.
@ -77,7 +77,7 @@ export class LibevShadowsocksServer implements ShadowsocksServer {
}));
return Promise.resolve(new LibevShadowsocksServerInstance(
childProcess, portNumber, password, encryptionMethod, accessUrl, statsSocket));
childProcess, portNumber, password, encryptionMethod, accessUrl, metricsSocket));
}
}
@ -88,7 +88,7 @@ class LibevShadowsocksServerInstance implements ShadowsocksInstance {
constructor(
private childProcess: child_process.ChildProcess, public portNumber: number, public password,
public encryptionMethod: string, public accessUrl: string,
private statsSocket: dgram.Socket) {}
private metricsSocket: dgram.Socket) {}
public stop() {
logging.info(`Stopping server on port ${this.portNumber}`);
@ -106,30 +106,30 @@ class LibevShadowsocksServerInstance implements ShadowsocksInstance {
// https://github.com/shadowsocks/shadowsocks-libev/blob/a16826b83e73af386806d1b51149f8321820835e/src/server.c#L172
public onInboundBytes(callback: (bytes: number, ipAddresses: string[]) => void) {
if (this.eventEmitter.listenerCount(this.INBOUND_BYTES_EVENT) === 0) {
this.createStatsListener();
this.createMetricsListener();
}
this.eventEmitter.on(this.INBOUND_BYTES_EVENT, callback);
}
private createStatsListener() {
private createMetricsListener() {
let lastInboundBytes = 0;
this.statsSocket.on('message', (buf: Buffer) => {
let statsMessage;
this.metricsSocket.on('message', (buf: Buffer) => {
let metricsMessage;
try {
statsMessage = parseStatsMessage(buf);
metricsMessage = parseMetricsMessage(buf);
} catch (err) {
logging.error('error parsing stats: ' + buf + ', ' + err);
logging.error('error parsing metrics: ' + buf + ', ' + err);
return;
}
if (statsMessage.portNumber !== this.portNumber) {
// Ignore stats for other ss-servers, which post to the same statsSocket.
if (metricsMessage.portNumber !== this.portNumber) {
// Ignore metrics for other ss-servers, which post to the same metricsSocket.
return;
}
const delta = statsMessage.totalInboundBytes - lastInboundBytes;
const delta = metricsMessage.totalInboundBytes - lastInboundBytes;
if (delta > 0) {
this.getConnectedClientIPAddresses()
.then((ipAddresses: string[]) => {
lastInboundBytes = statsMessage.totalInboundBytes;
lastInboundBytes = metricsMessage.totalInboundBytes;
this.eventEmitter.emit(this.INBOUND_BYTES_EVENT, delta, ipAddresses);
})
.catch((err) => {
@ -165,12 +165,12 @@ class LibevShadowsocksServerInstance implements ShadowsocksInstance {
}
}
interface StatsMessage {
interface MetricsMessage {
portNumber: number;
totalInboundBytes: number;
}
function parseStatsMessage(buf): StatsMessage {
function parseMetricsMessage(buf): MetricsMessage {
const jsonString = buf.toString()
.substr('stat: '.length) // remove leading "stat: "
.replace(/\0/g, ''); // remove trailing null terminator

View file

@ -23,28 +23,32 @@ import * as json_config from '../infrastructure/json_config';
import * as logging from '../infrastructure/logging';
import {LibevShadowsocksServer} from './libev_shadowsocks_server';
import {ManagerStats, ManagerStatsJson} from './manager_metrics';
import {ManagerMetrics, ManagerMetricsJson} from './manager_metrics';
import {bindService, ShadowsocksManagerService} from './manager_service';
import {createServerAccessKeyRepository} from './server_access_key';
import * as server_config from './server_config';
import {SharedStats, SharedStatsJson} from './shared_metrics';
import {SharedMetrics, SharedMetricsJson} from './shared_metrics';
const DEFAULT_STATE_DIR = '/root/shadowbox/persisted-state';
const MAX_STATS_FILE_AGE_MS = 5000;
interface StatsConfigJson {
// Serialized format for the metrics file.
// WARNING: Renaming fields will break backwards-compatibility.
interface MetricsConfigJson {
// Serialized ManagerStats object.
transferStats?: ManagerStatsJson;
transferStats?: ManagerMetricsJson;
// Serialized SharedStats object.
hourlyMetrics?: SharedStatsJson;
hourlyMetrics?: SharedMetricsJson;
}
function readStatsConfig(filename: string): json_config.JsonConfig<StatsConfigJson> {
const statsConfig = json_config.loadFileConfig<StatsConfigJson>(filename);
function readMetricsConfig(filename: string): json_config.JsonConfig<MetricsConfigJson> {
const metricsConfig = json_config.loadFileConfig<MetricsConfigJson>(filename);
// Make sure we have non-empty sub-configs.
statsConfig.data().transferStats = statsConfig.data().transferStats || {} as ManagerStatsJson;
statsConfig.data().hourlyMetrics = statsConfig.data().hourlyMetrics || {} as SharedStatsJson;
return new json_config.DelayedConfig(statsConfig, MAX_STATS_FILE_AGE_MS);
metricsConfig.data().transferStats =
metricsConfig.data().transferStats || {} as ManagerMetricsJson;
metricsConfig.data().hourlyMetrics =
metricsConfig.data().hourlyMetrics || {} as SharedMetricsJson;
return new json_config.DelayedConfig(metricsConfig, MAX_STATS_FILE_AGE_MS);
}
function main() {
@ -79,11 +83,11 @@ function main() {
const shadowsocksServer = new LibevShadowsocksServer(proxyHostname, verbose);
const statsConfig = readStatsConfig(getPersistentFilename('shadowbox_stats.json'));
const managerMetrics =
new ManagerStats(new json_config.ChildConfig(statsConfig, statsConfig.data().transferStats));
const sharedMetrics = new SharedStats(
new json_config.ChildConfig(statsConfig, statsConfig.data().hourlyMetrics), serverConfig,
const metricsConfig = readMetricsConfig(getPersistentFilename('shadowbox_stats.json'));
const managerMetrics = new ManagerMetrics(
new json_config.ChildConfig(metricsConfig, metricsConfig.data().transferStats));
const sharedMetrics = new SharedMetrics(
new json_config.ChildConfig(metricsConfig, metricsConfig.data().hourlyMetrics), serverConfig,
metricsUrl, new ip_location.MmdbLocationService());
logging.info('Starting...');

View file

@ -14,7 +14,7 @@
import {InMemoryConfig} from '../infrastructure/json_config';
import {ManagerStats, ManagerStatsJson} from './manager_metrics';
import {ManagerMetrics, ManagerMetricsJson} from './manager_metrics';
function addDays(baseDate: Date, days: number) {
const date = new Date(baseDate);
@ -22,26 +22,26 @@ function addDays(baseDate: Date, days: number) {
return date;
}
describe('ManagerStats', () => {
describe('ManagerMetrics', () => {
it('Saves traffic to config', (done) => {
const now = new Date();
const config = new InMemoryConfig({} as ManagerStatsJson);
const stats = new ManagerStats(config);
const config = new InMemoryConfig({} as ManagerMetricsJson);
const metrics = new ManagerMetrics(config);
let report = stats.get30DayByteTransfer();
let report = metrics.get30DayByteTransfer();
expect(report.bytesTransferredByUserId).toEqual({});
for (let di = 0; di < 40; di++) {
stats.recordBytesTransferred(addDays(now, -di), 'user-0', 1);
metrics.recordBytesTransferred(addDays(now, -di), 'user-0', 1);
}
report = stats.get30DayByteTransfer();
report = metrics.get30DayByteTransfer();
// This is being dropped
expect(report.bytesTransferredByUserId).toEqual({'user-0': 30});
// We are not cleaning this from the config.
expect(config.written.userIdSet).toEqual(['user-0']);
expect(Object.keys(config.written.dailyUserBytesTransferred).length).toEqual(40);
expect(new ManagerStats(new InMemoryConfig(config.written)).get30DayByteTransfer())
expect(new ManagerMetrics(new InMemoryConfig(config.written)).get30DayByteTransfer())
.toEqual(report);
done();
});

View file

@ -16,22 +16,24 @@ import {JsonConfig} from '../infrastructure/json_config';
import {AccessKeyId} from '../model/access_key';
import {DataUsageByUser} from '../model/metrics';
export interface ManagerStatsJson {
// Serialized format for the manager metrics.
// WARNING: Renaming fields will break backwards-compatibility.
export interface ManagerMetricsJson {
// Bytes per user per day. The key encodes the user+day in the form "userId-dateInYYYYMMDD".
dailyUserBytesTransferred?: Array<[string, number]>;
// Set of all User IDs for whom we have transfer stats.
// Set of all User IDs for whom we have transfer metrics.
// TODO: Delete userIdSet. It can be derived from dailyUserBytesTransferred.
userIdSet?: string[];
}
// ManagerStats keeps track of the number of bytes transferred per user, per day.
// ManagerMetrics keeps track of the number of bytes transferred per user, per day.
// Surfaced by the manager service to display on the Manager UI.
// TODO: Remove entries older than 30d.
export class ManagerStats {
export class ManagerMetrics {
private dailyUserBytesTransferred: Map<string, number>;
private userIdSet: Set<AccessKeyId>;
constructor(private config: JsonConfig<ManagerStatsJson>) {
constructor(private config: JsonConfig<ManagerMetricsJson>) {
const serializedObject = config.data();
if (serializedObject) {
this.dailyUserBytesTransferred = new Map(serializedObject.dailyUserBytesTransferred);
@ -73,7 +75,7 @@ export class ManagerStats {
// Returns the state of this object, e.g.
// {"dailyUserBytesTransferred":[["0-20170816",100],["1-20170816",100]],"userIdSet":["0","1"]}
private toJson(target: ManagerStatsJson) {
private toJson(target: ManagerMetricsJson) {
// Use [...] operator to serialize Map and Set objects to JSON.
target.dailyUserBytesTransferred = [...this.dailyUserBytesTransferred];
target.userIdSet = [...this.userIdSet];

View file

@ -19,7 +19,7 @@ import {JsonConfig} from '../infrastructure/json_config';
import * as logging from '../infrastructure/logging';
import {AccessKey, AccessKeyRepository} from '../model/access_key';
import {ManagerStats} from './manager_metrics';
import {ManagerMetrics} from './manager_metrics';
import {ServerConfigJson} from './server_config';
// Creates a AccessKey response.
@ -84,7 +84,7 @@ export class ShadowsocksManagerService {
private defaultServerName: string,
private serverConfig: JsonConfig<ServerConfigJson>,
private accessKeys: AccessKeyRepository,
private managerMetrics: ManagerStats,
private managerMetrics: ManagerMetrics,
) {}
public renameServer(req: RequestType, res: ResponseType, next: restify.Next): void {

View file

@ -71,7 +71,7 @@ class MockShadowsocksInstance implements ShadowsocksInstance {
export class MockShadowsocksServer {
startInstance(
portNumber: number, password: string, statsSocket: dgram.Socket,
portNumber: number, password: string, metricsSocket: dgram.Socket,
encryptionMethod?: string): Promise<ShadowsocksInstance> {
const mock = new MockShadowsocksInstance(portNumber, password, encryptionMethod);
return Promise.resolve(mock);

View file

@ -22,8 +22,8 @@ import {AccessKey, AccessKeyId, AccessKeyRepository} from '../model/access_key';
import {ShadowsocksInstance, ShadowsocksServer} from '../model/shadowsocks_server';
import {TextFile} from '../model/text_file';
import {ManagerStats} from './manager_metrics';
import {SharedStats} from './shared_metrics';
import {ManagerMetrics} from './manager_metrics';
import {SharedMetrics} from './shared_metrics';
// The format as json of access keys in the config file.
interface AccessKeyConfig {
@ -90,16 +90,16 @@ class AccessKeyConfigFile {
export function createServerAccessKeyRepository(
proxyHostname: string, textFile: TextFile, shadowsocksServer: ShadowsocksServer,
managerMetrics: ManagerStats, sharedMetrics: SharedStats): Promise<AccessKeyRepository> {
managerMetrics: ManagerMetrics, sharedMetrics: SharedMetrics): Promise<AccessKeyRepository> {
const configFile = new AccessKeyConfigFile(textFile);
const configJson = configFile.loadConfig();
const reservedPorts = getReservedPorts(configJson.accessKeys);
// Create and save the stats socket.
return createBoundUdpSocket(reservedPorts).then((statsSocket) => {
reservedPorts.add(statsSocket.address().port);
// Create and save the metrics socket.
return createBoundUdpSocket(reservedPorts).then((metricsSocket) => {
reservedPorts.add(metricsSocket.address().port);
return new ServerAccessKeyRepository(
proxyHostname, configFile, configJson, shadowsocksServer, statsSocket, managerMetrics,
proxyHostname, configFile, configJson, shadowsocksServer, metricsSocket, managerMetrics,
sharedMetrics);
});
}
@ -129,8 +129,8 @@ class ServerAccessKeyRepository implements AccessKeyRepository {
constructor(
private proxyHostname: string, private configFile: AccessKeyConfigFile,
private configJson: ConfigJson, private shadowsocksServer: ShadowsocksServer,
private statsSocket: dgram.Socket, private managerMetrics: ManagerStats,
private sharedMetrics: SharedStats) {
private metricsSocket: dgram.Socket, private managerMetrics: ManagerMetrics,
private sharedMetrics: SharedMetrics) {
for (const accessKeyJson of this.configJson.accessKeys) {
this.startInstance(accessKeyJson).catch((error) => {
logging.error(`Failed to start Shadowsocks instance for key ${accessKeyJson.id}: ${error}`);
@ -203,7 +203,7 @@ class ServerAccessKeyRepository implements AccessKeyRepository {
private startInstance(accessKeyJson: AccessKeyConfig): Promise<void> {
return this.shadowsocksServer
.startInstance(
accessKeyJson.port, accessKeyJson.password, this.statsSocket,
accessKeyJson.port, accessKeyJson.password, this.metricsSocket,
accessKeyJson.encryptionMethod)
.then((ssInstance) => {
ssInstance.onInboundBytes(

View file

@ -16,6 +16,8 @@ import * as uuidv4 from 'uuid/v4';
import * as json_config from '../infrastructure/json_config';
// Serialized format for the server config.
// WARNING: Renaming fields will break backwards-compatibility.
export interface ServerConfigJson {
serverId: string;
metricsEnabled: boolean;

View file

@ -15,7 +15,7 @@
import * as https from 'https';
import * as ip_location from '../infrastructure/ip_location';
import {PerUserStats} from '../model/metrics';
import {PerUserMetrics} from '../model/metrics';
import * as shared_metrics from './shared_metrics';
@ -185,7 +185,7 @@ describe('getHourlyServerMetricsReport', () => {
});
});
function getPerUserStats(ipAddresses: string[]): PerUserStats {
function getPerUserStats(ipAddresses: string[]): PerUserMetrics {
return {bytesTransferred: 123, anonymizedIpAddresses: new Set(ipAddresses)};
}

View file

@ -19,13 +19,15 @@ import * as ip_location from '../infrastructure/ip_location';
import {JsonConfig} from '../infrastructure/json_config';
import * as logging from '../infrastructure/logging';
import {AccessKeyId} from '../model/access_key';
import {PerUserStats} from '../model/metrics';
import {PerUserMetrics} from '../model/metrics';
import {LastHourMetricsReadyCallback} from '../model/metrics';
import * as ip_util from './ip_util';
import {ServerConfigJson} from './server_config';
export interface SharedStatsJson {
// Serialized format for the shared metrics.
// WARNING: Renaming fields will break backwards-compatibility.
export interface SharedMetricsJson {
startTimestamp?: number;
// TODO: Save the countries rather than anonymized IPs. There's no point in keeping the IPs.
lastHourUserStatsObj?:
@ -34,31 +36,31 @@ export interface SharedStatsJson {
const LAST_HOUR_METRICS_READY_EVENT = 'lastHourMetricsReady';
// Keeps track of the connection stats per user, since the startDatetime.
// Keeps track of the connection metrics per user, since the startDatetime.
// This is reported to the Outline team if the admin opts-in.
export class SharedStats {
export class SharedMetrics {
private eventEmitter = new events.EventEmitter();
// Date+time at which we started recording connection stats, e.g.
// Date+time at which we started recording connection metrics, e.g.
// in case this object is constructed from data written to disk.
public startDatetime: Date;
// Map from the metrics AccessKeyId to stats (bytes transferred, IP addresses).
public lastHourUserStats: Map<AccessKeyId, PerUserStats>;
// Map from the metrics AccessKeyId to metrics (bytes transferred, IP addresses).
public lastHourUserMetrics: Map<AccessKeyId, PerUserMetrics>;
constructor(
private config: JsonConfig<SharedStatsJson>,
private config: JsonConfig<SharedMetricsJson>,
private serverConfig: JsonConfig<ServerConfigJson>, metricsUrl: string,
ipLocationService: ip_location.IpLocationService) {
const serializedObject = this.config.data();
this.startDatetime =
serializedObject.startTimestamp ? new Date(serializedObject.startTimestamp) : new Date();
this.lastHourUserStats = new Map<AccessKeyId, PerUserStats>();
this.lastHourUserMetrics = new Map<AccessKeyId, PerUserMetrics>();
if (serializedObject.lastHourUserStatsObj) {
Object.keys(serializedObject.lastHourUserStatsObj).map((userId) => {
const perUserStatsObj = serializedObject.lastHourUserStatsObj[userId];
this.lastHourUserStats.set(userId, {
this.lastHourUserMetrics.set(userId, {
bytesTransferred: perUserStatsObj.bytesTransferred,
anonymizedIpAddresses: new Set(perUserStatsObj.anonymizedIpAddresses)
});
@ -86,20 +88,20 @@ export class SharedStats {
// CONSIDER: accepting hashedIpAddresses, which can be persisted to disk
// and reported to the metrics server (to approximate number of devices per userId).
recordBytesTransferred(userId: AccessKeyId, numBytes: number, ipAddresses: string[]) {
const perUserStats = this.lastHourUserStats.get(userId) ||
const perUserMetrics = this.lastHourUserMetrics.get(userId) ||
{bytesTransferred: 0, anonymizedIpAddresses: new Set<string>()};
perUserStats.bytesTransferred += numBytes;
perUserMetrics.bytesTransferred += numBytes;
const anonymizedIpAddresses = getAnonymizedAndDedupedIpAddresses(ipAddresses);
for (const ip of anonymizedIpAddresses) {
perUserStats.anonymizedIpAddresses.add(ip);
perUserMetrics.anonymizedIpAddresses.add(ip);
}
this.lastHourUserStats.set(userId, perUserStats);
this.lastHourUserMetrics.set(userId, perUserMetrics);
this.toJson(this.config.data());
this.config.write();
}
reset(): void {
this.lastHourUserStats = new Map<AccessKeyId, PerUserStats>();
this.lastHourUserMetrics = new Map<AccessKeyId, PerUserMetrics>();
this.startDatetime = new Date();
this.toJson(this.config.data());
this.config.write();
@ -117,11 +119,11 @@ export class SharedStats {
// Returns the state of this object, e.g.
// {"startTimestamp":1502896650353,"lastHourUserStatsObj":{"0":{"bytesTransferred":100,"anonymizedIpAddresses":["2620:0:1003:0:0:0:0:0","5.2.79.0"]}}}
private toJson(target: SharedStatsJson) {
private toJson(target: SharedMetricsJson) {
// lastHourUserStats is a Map containing Set structures. Convert to an object
// with array values.
const lastHourUserStatsObj = {};
this.lastHourUserStats.forEach((perUserStats, userId) => {
this.lastHourUserMetrics.forEach((perUserStats, userId) => {
lastHourUserStatsObj[userId] = {
bytesTransferred: perUserStats.bytesTransferred,
anonymizedIpAddresses: [...perUserStats.anonymizedIpAddresses]
@ -133,17 +135,17 @@ export class SharedStats {
}
private generateHourlyReport(): void {
if (this.lastHourUserStats.size === 0) {
// No connection stats to report.
if (this.lastHourUserMetrics.size === 0) {
// No connection metrics to report.
return;
}
this.eventEmitter.emit(
LAST_HOUR_METRICS_READY_EVENT, this.startDatetime,
new Date(), // endDatetime is the current date and time.
this.lastHourUserStats);
this.lastHourUserMetrics);
// Reset connection stats to begin recording the next hour.
// Reset connection metrics to begin recording the next hour.
this.reset();
}
}
@ -162,16 +164,16 @@ function getAnonymizedAndDedupedIpAddresses(ipAddresses: string[]): Set<string>
export function getHourlyServerMetricsReport(
serverId: string, startDatetime: Date, endDatetime: Date,
lastHourUserStats: Map<AccessKeyId, PerUserStats>,
lastHourUserMetrics: Map<AccessKeyId, PerUserMetrics>,
ipLocationService: ip_location.IpLocationService): Promise<HourlyServerMetricsReport|null> {
if (lastHourUserStats.size === 0) {
// Stats are empty, no need to post a report
if (lastHourUserMetrics.size === 0) {
// Metrics are empty, no need to post a report
return Promise.resolve(null);
}
// convert lastHourUserStats to an array HourlyUserMetricsReport
const userReportPromises = [];
lastHourUserStats.forEach((perUserStats, userId) => {
userReportPromises.push(getHourlyUserMetricsReport(userId, perUserStats, ipLocationService));
lastHourUserMetrics.forEach((perUserMetrics, userId) => {
userReportPromises.push(getHourlyUserMetricsReport(userId, perUserMetrics, ipLocationService));
});
return Promise.all(userReportPromises).then((userReports: HourlyUserMetricsReport[]) => {
// Remove any userReports containing sanctioned countries, and return
@ -222,10 +224,10 @@ interface HourlyUserMetricsReport {
}
function getHourlyUserMetricsReport(
userId: AccessKeyId, perUserStats: PerUserStats,
userId: AccessKeyId, perUserMetrics: PerUserMetrics,
ipLocationService: ip_location.IpLocationService): Promise<HourlyUserMetricsReport> {
const countryPromises = [];
for (const ip of perUserStats.anonymizedIpAddresses) {
for (const ip of perUserMetrics.anonymizedIpAddresses) {
const countryPromise = ipLocationService.countryForIp(ip).catch((e) => {
logging.warn(`Failed countryForIp call: ${e}`);
return 'ERROR';
@ -235,7 +237,7 @@ function getHourlyUserMetricsReport(
return Promise.all(countryPromises).then((countries: string[]) => {
return {
userId,
bytesTransferred: perUserStats.bytesTransferred,
bytesTransferred: perUserMetrics.bytesTransferred,
countries: getWithoutDuplicates(countries)
};
});