From a7641eefb2055dc4b2188d2e16f84e4066fd0f87 Mon Sep 17 00:00:00 2001 From: fortuna Date: Tue, 11 Sep 2018 15:04:04 -0400 Subject: [PATCH] Rename stats to metrics --- coreos.txt | 52 ++++++++++++++++ key.txt | 51 ++++++++++++++++ src/shadowbox/infrastructure/json_config.ts | 2 +- src/shadowbox/model/metrics.ts | 8 +-- src/shadowbox/model/shadowsocks_server.ts | 2 +- .../server/libev_shadowsocks_server.ts | 34 +++++------ src/shadowbox/server/main.ts | 34 ++++++----- src/shadowbox/server/manager_metrics.spec.ts | 16 ++--- src/shadowbox/server/manager_metrics.ts | 14 +++-- src/shadowbox/server/manager_service.ts | 4 +- src/shadowbox/server/mocks/mocks.ts | 2 +- src/shadowbox/server/server_access_key.ts | 20 +++---- src/shadowbox/server/server_config.ts | 2 + src/shadowbox/server/shared_metrics.spec.ts | 4 +- src/shadowbox/server/shared_metrics.ts | 60 ++++++++++--------- 15 files changed, 209 insertions(+), 96 deletions(-) create mode 100644 coreos.txt create mode 100644 key.txt diff --git a/coreos.txt b/coreos.txt new file mode 100644 index 00000000..e9d14d5e --- /dev/null +++ b/coreos.txt @@ -0,0 +1,52 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAy8RtwobMKZCcvVBkP6oQqb0moUJEv1N2G166p/fr0DjoK8Ei +70xjKGcqDFhXvR75oW4DjSL1jUwSXfI4L8b3Kd71UBqJD2Xpb6upLII6MxIxnanU +hI0a0R7x8G64GgC204YmMB7HN2Z8rxGrC7H0qDCUbn+vKEBSfSxp99XRgRBzk1Z0 +sNPGd0xxDh8Gsm7ViGuDmyEv5ICCrAMVr2vEjyHczqb9FQe7/7JSKrY3rrOFcH0k +V0wtw9nT5wp9uJGxO0EYy5AYma7CKeUC46ijLB16rbNc22FaezsMTHbRnXpks+KT +0Pt8DLO5tjQsZ4yo5vZDb9WzWuMdUkkBIHw/9gUP3Hb+OKjdsdjc6A+sfKXrotBA +UdirlMTs2FzRpDJhAg9qTL/mngJtEBN7wXLhXKPXCZdtH/dBX91QZBbIxNaqeY/A +AKG6VZONGgZcVtU1ix8+uqijABkWprhVLO0jBxkSPZiMy2iTz6p+FJ+zy/ZbRfl+ +YZGjhs6xdklhll1zrCSXBcY2aAUsgQWwZl8hG1ibTcFGq0lxCRdcXeYwX8uuRFwE +ZFBq3yG6SZOR4JZLWrEgLHdb6T7rik2U73cyX+/3x+j2QG65mvqLdisaja3d/961 +Tjj4qFliz5ZWPECVWNigqnmqSe/nv5XPC2A93GM25PHFY+bJlpaaq0QWP1ECAwEA +AQKCAgBZgFP9p5uxfhV6if8ef6KGC9EV77emmhA8gWVXNexcL0K7RUAE//Zl3rp3 +Za5UIXDgWSQyL/LPN2Sx4xyOz8PsnkP+BUnCe68HH81VAXZyzEEC0X/JIPlwdTkr +tFYlBb9INZo8dKhoSxnlA8uvfWDLJ1trFaZn9ajF1mZNN5uoJwO86bKjoMGB0Q+v +di1I3qnoG+FYmEEjCtdamphBzwItJGCKXIq5XAZVj4vLuvHGSJAKEs2NkqZfaiRL +TS5fjY7dSgCMGSTVDA+4uyCDwqS5UdF6zlew/JfznMIQK/hyRTpKUPFAT9Xy9lZS +E6SVbxEZMX35d1IqT6unYu2dyTWGHolUNieFnUCZsXCyOl7lqY0OGSEMAsd+kFDD +LMDkq/7QffJhQve2Av5kwKuPeUt/wzwJ+RPVawxHOPZNwcA7qeGzTcClgUu+1LMo +jxuLbV9pcMfv07fD2Rf5Cm2EW3lY5aj0/D45sia6IjMYrwUzFHztmUcvO/hj6Kur +Jmbyuw1Dn44StXWYrSef0aWVqmKkWh8NY03mYuLFOEhkvaaAj+dGjX3gIcqAbC+f +GjCXb7vWGL/RanNVmaZVu3W9zTURHZXbeKU7mUZ7y5gug1mIcfaONKsQ9I/eiZaP +uZmFfrePjzvIlBvG5mVhdm8U9DC1PlEz9VxYDFy/j08Nkx0F4QKCAQEA/Tm90Gr6 +e63kZgX+KraFdnCnoDJF2WaiDRAmDikc4tFwaAcahp84Z6Z/OkYw3fq4786rsusf +QBqACg4mKl0V//atGNBCPc0rResHkOpqW7+mwpxT28hvL6zrFRJiRqbG5MSIkL/A +2i/f6po7g8voye4gHtdRHPKdERJ8Juzdm2ZdTLSo0XOT6rlvwECHHvEXlHfugHDd +zNEqJfpwU6E/nU4o6FTOO3xZw2SOnCNdR/Am3xdJFHRFIrZ1/0sGv5EU+HUtDfEq +9jmmWaZYF2pjCIT82EQ00tNpBvwQZ54aQk/xukxbCGR+nycJ9cMl3RbzumnZTtcj +WiMw+rVOUFrWJQKCAQEAzf/265/bAqPC+NbGOR28WBAI2Ja+QRVgEiJaSVt7ngDm +VAhVF0fHjFRX+c8YjR6fKUPHzQOAV80xWN/k9JEg63b58whm/QVV+5WcQ/9yD2SB +6YtBimzT40c+b39Y7tNLVGQ8n512jVHGKBuHjqWmXVI37FxkbtxHBWSaiCz2EVyY +BvMmsaHjCYgr2cNnOmwJjsjBLqDR1mZrds1LqYu+kM6THLTauPhj+i3cCH3ZH2gU +0EG//S14WMnZuBrVv6BPzIeO09PwFEE7KzPSSwoB9H0HH3lIyQxoYLWr2OOBDz9s +kmQ/piqSbtO+ARWYd86oWjLjWF4G/HvjVzeU9cWuvQKCAQEAsDIaOkgFrEMt9jNb +TBseOHBgop34bjH0tgQzhozi1YwHm8q9kUh+ddirFEA8xmgrgGkcnWzunKsTxmtb +8QQ+R5E7llVqkhgNcSP9ar9BbD+paCZgT0Bi5Rh7pnjZOvHW2N1LbPSP2wGO222f +1a/vdXokjXEitnK2CWgETQ1pkTSj3Lij8sFp/dwzvuDnZAc7cgoVQPfHzTkJC837 +lKVRX0JAQpCnw0peJw/0Dv6obGLUmUxZhEr2xBWTeySYOHlZzxuxUs8pJpSshBqz +lu9mo0ntqQmke8GwhbSkMYUYHmYD+64fdXJ/jHwceQ3lbbYHtwDpvOsDZTexX/EB +4fWipQKCAQAVT7G/1p3VvBNjovSG3CisA5ymq5GrMgbqWVt1010Kj5VEhEgpTGe8 +gM1JLr+fedeFcVmuP/p7GuNMCn2pP4pkUb6yAeCFtJOcn3G3JyoppYA7JQj2xSN2 +k9xFtKsCqiFU7bnH2YZ2QEt7wr1XaJO5e9QFQ9mwDmHakPnbtKsQSMABmA4cul6+ +kbPXp3t6c8rZVrOFm2WaBKaBd+On+qkQWg6mHZ+zGx9ctnnY9wwLT703flXaX2Xf +6aH4he4vEOqwgiWojHh93/G7GnVbBgIFxRmDjAyXoAz8VE8e9QpZBXq5+l0LV8Qm +awlxG0bWvi50hmc2sSOP41E1qK5kbrsRAoIBAQDUDpgJ2HrCKnkIFj3IzPHnw/Ur ++bmWUtfCCP07THH7dYMSfZLMwXwwJeNHtJIo4TUqghcTSH+QJYhCwOkwwexa/F4Z +E3FRy/KBlsjIcL/bEelCK8PpsZd9fYZkrHK+hz7uyiXULrErbkiQo0akze1DghCw +R0gSgkRj2YLZ+2e5Tud/eDC1l9JbVWuAOj1n2/MdxsRIMSMBdTRaivLHuWVi83Gx +yyGuJxy/H9Z0izmDJzbPl3ETf6OLLvJY3tL8qgQrQWajvu8dvGyi2F4KpuaLu29b +pTg7GLNkzLcltLZpcJf9sk0+XJP1Ky07Cqg0qfVSBIzyCQ5XQgAnXy67lOWK +-----END RSA PRIVATE KEY----- + diff --git a/key.txt b/key.txt new file mode 100644 index 00000000..c1936468 --- /dev/null +++ b/key.txt @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAnrDtsh/DFaawDjY1XOijPpk1dpJlcKibSLfrTeGJAepT1IS2 +h8s/V1PP4WOqCpHAoukzcJFbyIhAFUsuXlpY32YEAKIHA0dEHvdq/GgluylyZqLv +2r2II+Fm01142ZXQGg+SEI4i0+fkZsWU7HnbmBydJKFpyn5yQv3LMwhaPCZmg5B4 +SxnkeigfrmxfKP2xZ9qoxNivVulf0mlcRl/UkjXqFUCAR6zCUqiMIdZ/JYiaMd+3 +39owVqiTBo2+AdtiO4GxQ964NwqcfL7cdhHslKt8/MUsOuTkAuMWT41SiwUih6nk +E87rgwQbg4Iv70YEcqR6d0JR8VWj8pfPnSTECN4XYFdkE59udS5q6R95v2VF2EKL +MV4R5S+DpXraVtg5qxkTBXXQACfoKOr/PDCG3SbAJylORHML7JMPpN4Ovt2cft8x +Fkrrv9bCL8duMHSrVd+jlcYb7KWB8kDMSIkI6j6Vgj+FsuHGwTzAHdwikrCNvX2z +SH+hhz8QuzOb7HyTCFRBeueb3AMm5a3UAn1BHdsY+sU/8hqF3VUHx6Eh0A1wAShv +kU2S8w4XkFsiam0fJ9pPes/UIvYCsOpkq/Cyhxxv1DW+3iqDqT3cbvJArNN16iyT +CZMeejkXUP61dQKevc8wbtHHTsN1HZAagIJZGOTiJmErHv8X4w5U5SIRBEUCAwEA +AQKCAgAMUy2C4hiiu9l8oRUv7BmRqRCP7vrV7yJwWWh7GgMsFOkB3nTWwyBeRfIl +TTpNq9hMTtRh78gzIPHpNXeaeGXio+e3rN3ikUxnI0w54lTb3nI2Kn16fbHvJ3h2 +/hF+xLXu8Dql8oQd9Sq2GK5iV2yIueAykh7HTV7OeSupAQMRHOJlYXkWTCKEok0j +nOMfKeT3bfIhp7qmg2Wfz/MMvDCkUm+lyuarqm1FQjXwAtrJLXzcVsXPKwEKGfmh +TqztM/7DJGWCIfAoxjg1MBWrTeUT/uWsNBwlTtWuq7h14UIB7hoqczV1nGKKQG1q +n1fdu5Bo3wFr9cird71OACBt0gcvkctVejgwTNg9z/M9dLFvTguty2Hwbb1wCg8F +VxKXhDxUxpcl+1MQKqPSxWcSv7v6Zhtdj5sEwPF3Q2MUV51geKP1Ds/0OrOr88Lb +J1TaD53lFe5TIHM70CVswy+Nn64y7f/7zDi5OPabmYEkyUcu0O3aPIEVHxf3FwZD +/G5nMs4sRoaFGuMiOIbXg0Gw6MJ9eQTdWxFSK27P0gncsDZrtu4pi87i3Rt4917T +5pIrduUsqIILbpyo17pvpufkXPrukgEHjbWejl3s1wleWVktWb7D45mppmXaSuc7 +aM++O43W97DWnmSCbhoD/y8dF8bJ3f65ekkzNeRUuUU2zdhxgQKCAQEA0W4Qg/Z5 +hnBROWsZNddiiEWUAASYgMGdtPKj7CYovVhL+1zX0wHp371k0TyOwbTh+Pg+S5z9 +rP+9/yltMyx5dToVF20zsCmfVPzGl1ksIiiaJfy70IPTVT2bj5sFgv+bnTRibXgN +LDyJpCUR4Wfzl8VBgqUYSyZc6WVH35SKvudfMgvOKtHkb56Pk5gRI3O5VyDUHL4g +8k1HCOoVxR1w5vZkSmImQaI2jLZ0vl6F1w1SlADci/jyNRV/3y7frAYH73v2pKJ3 +iTgF8+cZXsgHP9UkSHA0CXM/h62DVh1iWRXEyzWdGe9roivmPbAo//mGtYLhHGge +kT1oycZ8DPjDcQKCAQEAwfqGKqfq62YEKUC44YrTLrsRtepKa8FAkD2MEHeyPOMh +nA5ECyoNJKjViqEsb8og5USOfOV223ZLGis8Qnx4Z9wkO/7g9ky0zeBsZbIrBpFi +T1vCLmi1Qh9GqvCkiIJ8lvkGQq/tCvLnBLOtM/dFqs4oxns++5fsuWLf7gL5eDHk +pb2W8HrZHWMdxHiwv+5GD8WJI35vcmgaVBQaKrkIrRHhkklvrJCRtmZ5y1WlKUzm +ywy8NSphuJ0nUdjKHKy4dufq6HbRQWjZQczT4VnujuT9syvsxVfoEjRGKZeeNw+I ++4o+fnNbZ3mYOtJWW6cB5oQDIWu3JSZ+Anjhb0K8FQKCAQEAzX4yxFkm+uvgNvfI +P9U8ADxNMQtRXB0eknr2rvLuTIOD4ntB5fBdu8TJVKkX5ieHBtUFwwmiu4ogsmrC +lFDSSF0abucerX7ZsPlqHv1HWaj/P8DRxJJk3aHarrjMWrJVzZWl8oW2Xy5zW9Vn +ywVFtii90+QMh0h4KCbRtCa7UQATnzaIL+nNPFyXwpmWT3PwavZySlWgXD/JMI0H +mWb+7hDbbUULBqGU5tLskBKNPur6mPCTduBpP/79fk8u90rfpHO9GeO0aLbI2H5s +nVymCFMqC83UsWUc9BMj6G5insjGVSIhKV6L/Q8YFnVwdWIwdI+cNFRRke9wj3Or +Kss4cQKCAQAEvFkKMY9Kr/LqDup0ly8QtQB5sH6gotcwrk/9Fu8DDYiEhtSicSRh +AL415DlxgT3MWyAfbHq6YOj0epm+BcvqvTUlQdO8L6M6Y4BB+1eRkXsU9OiIuYWz +V5AiHD3oF0dzaCD+/8yJt+Rr+PcBjcflo6LbNacT/WGKJR/Sb8AnbxBl+3rz5Avo +68KOSWQHS4nqWKhAdZXC9UevRc5dvKa5kvYu3Bwd2mm0Skwu6qhdmcMIsgbmRWKd +XzjWhrRofs8CGCdkBYKWVjj3okiJ9+gbFPwco7XkG4FO8HfGDC2QqpBtk7Jy494X +aKCOzxPMqQci2ZY5+qc+APKSnODkFn0BAoIBAEeov1NmKjF9YLzDnCYKpEzT0/CS +K8Ipcq3B5M1wOanrv9lFdSZdmdqY5XWTcSgjNfE75kALUstiUoUVBxmIZY8b/I2t +Vun3shvoHv1Bxj8qKnOPBG93lUC6Xs3/9HrvmXJGPk+keMiglFLPNFmZWV9ElkIv +6Q1E2kE3oGFUmhfQAqJ302QrHnIvcocfrxtsN3rh7s0NTrsadEJunzeW1C8SYXLO +gZtvv5Iw4ulgQD7f3Y7VbUlqarWztLiuGujjlts/jkqMCp+dgWEMPjGxxnyVHk3a +0MQPWJrhqHxBq25tAxu3Gd33WDOPL9wceyyVnpvIUVj7zU35h/sJUM7aHrg= +-----END RSA PRIVATE KEY----- diff --git a/src/shadowbox/infrastructure/json_config.ts b/src/shadowbox/infrastructure/json_config.ts index 4e0d0ba8..d4cd3384 100644 --- a/src/shadowbox/infrastructure/json_config.ts +++ b/src/shadowbox/infrastructure/json_config.ts @@ -47,7 +47,7 @@ export class FileConfig implements JsonConfig { write() { // Write to temporary file, then move that temporary file to the - // persistent location, to avoid accidentally breaking the stats file. + // persistent location, to avoid accidentally breaking the metrics file. // Use *Sync calls for atomic operations, to guard against corrupting // these files. const tempFilename = `${this.filename}.${Date.now()}`; diff --git a/src/shadowbox/model/metrics.ts b/src/shadowbox/model/metrics.ts index 2cedafae..e572811e 100644 --- a/src/shadowbox/model/metrics.ts +++ b/src/shadowbox/model/metrics.ts @@ -15,15 +15,15 @@ import {AccessKeyId} from './access_key'; export type LastHourMetricsReadyCallback = - (startDatetime: Date, endDatetime: Date, lastHourUserStats: Map) => - void; + (startDatetime: Date, endDatetime: Date, + lastHourUserMetrics: Map) => void; -export interface PerUserStats { +export interface PerUserMetrics { bytesTransferred: number; anonymizedIpAddresses: Set; } -// Byte transfer stats for the past 30 days, including both inbound and outbound. +// Byte transfer metrics for the past 30 days, including both inbound and outbound. // TODO: this is copied at src/model/server.ts. Both copies should // be kept in sync, until we can find a way to share code between the web_app // and shadowbox. diff --git a/src/shadowbox/model/shadowsocks_server.ts b/src/shadowbox/model/shadowsocks_server.ts index c30e99e3..f2fdfe04 100644 --- a/src/shadowbox/model/shadowsocks_server.ts +++ b/src/shadowbox/model/shadowsocks_server.ts @@ -16,7 +16,7 @@ import * as dgram from 'dgram'; export interface ShadowsocksServer { startInstance( - portNumber: number, password: string, statsSocket: dgram.Socket, + portNumber: number, password: string, metricsSocket: dgram.Socket, encryptionMethod?: string): Promise; } diff --git a/src/shadowbox/server/libev_shadowsocks_server.ts b/src/shadowbox/server/libev_shadowsocks_server.ts index c16be248..a983289c 100644 --- a/src/shadowbox/server/libev_shadowsocks_server.ts +++ b/src/shadowbox/server/libev_shadowsocks_server.ts @@ -30,17 +30,17 @@ export class LibevShadowsocksServer implements ShadowsocksServer { constructor(private publicAddress: string, private verbose: boolean) {} public startInstance( - portNumber: number, password: string, statsSocket: dgram.Socket, + portNumber: number, password: string, metricsSocket: dgram.Socket, encryptionMethod = this.DEFAULT_METHOD): Promise { logging.info(`Starting server on port ${portNumber}`); - const statsAddress = statsSocket.address(); + const metricsAddress = metricsSocket.address(); const commandArguments = [ '-m', encryptionMethod, // Encryption method '-u', // Allow UDP '--fast-open', // Allow TCP fast open '-p', portNumber.toString(), '-k', password, '--manager-address', - `${statsAddress.address}:${statsAddress.port}` + `${metricsAddress.address}:${metricsAddress.port}` ]; logging.info('starting ss-server with args: ' + commandArguments.join(' ')); // Add the system DNS servers. @@ -77,7 +77,7 @@ export class LibevShadowsocksServer implements ShadowsocksServer { })); return Promise.resolve(new LibevShadowsocksServerInstance( - childProcess, portNumber, password, encryptionMethod, accessUrl, statsSocket)); + childProcess, portNumber, password, encryptionMethod, accessUrl, metricsSocket)); } } @@ -88,7 +88,7 @@ class LibevShadowsocksServerInstance implements ShadowsocksInstance { constructor( private childProcess: child_process.ChildProcess, public portNumber: number, public password, public encryptionMethod: string, public accessUrl: string, - private statsSocket: dgram.Socket) {} + private metricsSocket: dgram.Socket) {} public stop() { logging.info(`Stopping server on port ${this.portNumber}`); @@ -106,30 +106,30 @@ class LibevShadowsocksServerInstance implements ShadowsocksInstance { // https://github.com/shadowsocks/shadowsocks-libev/blob/a16826b83e73af386806d1b51149f8321820835e/src/server.c#L172 public onInboundBytes(callback: (bytes: number, ipAddresses: string[]) => void) { if (this.eventEmitter.listenerCount(this.INBOUND_BYTES_EVENT) === 0) { - this.createStatsListener(); + this.createMetricsListener(); } this.eventEmitter.on(this.INBOUND_BYTES_EVENT, callback); } - private createStatsListener() { + private createMetricsListener() { let lastInboundBytes = 0; - this.statsSocket.on('message', (buf: Buffer) => { - let statsMessage; + this.metricsSocket.on('message', (buf: Buffer) => { + let metricsMessage; try { - statsMessage = parseStatsMessage(buf); + metricsMessage = parseMetricsMessage(buf); } catch (err) { - logging.error('error parsing stats: ' + buf + ', ' + err); + logging.error('error parsing metrics: ' + buf + ', ' + err); return; } - if (statsMessage.portNumber !== this.portNumber) { - // Ignore stats for other ss-servers, which post to the same statsSocket. + if (metricsMessage.portNumber !== this.portNumber) { + // Ignore metrics for other ss-servers, which post to the same metricsSocket. return; } - const delta = statsMessage.totalInboundBytes - lastInboundBytes; + const delta = metricsMessage.totalInboundBytes - lastInboundBytes; if (delta > 0) { this.getConnectedClientIPAddresses() .then((ipAddresses: string[]) => { - lastInboundBytes = statsMessage.totalInboundBytes; + lastInboundBytes = metricsMessage.totalInboundBytes; this.eventEmitter.emit(this.INBOUND_BYTES_EVENT, delta, ipAddresses); }) .catch((err) => { @@ -165,12 +165,12 @@ class LibevShadowsocksServerInstance implements ShadowsocksInstance { } } -interface StatsMessage { +interface MetricsMessage { portNumber: number; totalInboundBytes: number; } -function parseStatsMessage(buf): StatsMessage { +function parseMetricsMessage(buf): MetricsMessage { const jsonString = buf.toString() .substr('stat: '.length) // remove leading "stat: " .replace(/\0/g, ''); // remove trailing null terminator diff --git a/src/shadowbox/server/main.ts b/src/shadowbox/server/main.ts index 9e0af712..5927ca74 100644 --- a/src/shadowbox/server/main.ts +++ b/src/shadowbox/server/main.ts @@ -23,28 +23,32 @@ import * as json_config from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; import {LibevShadowsocksServer} from './libev_shadowsocks_server'; -import {ManagerStats, ManagerStatsJson} from './manager_metrics'; +import {ManagerMetrics, ManagerMetricsJson} from './manager_metrics'; import {bindService, ShadowsocksManagerService} from './manager_service'; import {createServerAccessKeyRepository} from './server_access_key'; import * as server_config from './server_config'; -import {SharedStats, SharedStatsJson} from './shared_metrics'; +import {SharedMetrics, SharedMetricsJson} from './shared_metrics'; const DEFAULT_STATE_DIR = '/root/shadowbox/persisted-state'; const MAX_STATS_FILE_AGE_MS = 5000; -interface StatsConfigJson { +// Serialized format for the metrics file. +// WARNING: Renaming fields will break backwards-compatibility. +interface MetricsConfigJson { // Serialized ManagerStats object. - transferStats?: ManagerStatsJson; + transferStats?: ManagerMetricsJson; // Serialized SharedStats object. - hourlyMetrics?: SharedStatsJson; + hourlyMetrics?: SharedMetricsJson; } -function readStatsConfig(filename: string): json_config.JsonConfig { - const statsConfig = json_config.loadFileConfig(filename); +function readMetricsConfig(filename: string): json_config.JsonConfig { + const metricsConfig = json_config.loadFileConfig(filename); // Make sure we have non-empty sub-configs. - statsConfig.data().transferStats = statsConfig.data().transferStats || {} as ManagerStatsJson; - statsConfig.data().hourlyMetrics = statsConfig.data().hourlyMetrics || {} as SharedStatsJson; - return new json_config.DelayedConfig(statsConfig, MAX_STATS_FILE_AGE_MS); + metricsConfig.data().transferStats = + metricsConfig.data().transferStats || {} as ManagerMetricsJson; + metricsConfig.data().hourlyMetrics = + metricsConfig.data().hourlyMetrics || {} as SharedMetricsJson; + return new json_config.DelayedConfig(metricsConfig, MAX_STATS_FILE_AGE_MS); } function main() { @@ -79,11 +83,11 @@ function main() { const shadowsocksServer = new LibevShadowsocksServer(proxyHostname, verbose); - const statsConfig = readStatsConfig(getPersistentFilename('shadowbox_stats.json')); - const managerMetrics = - new ManagerStats(new json_config.ChildConfig(statsConfig, statsConfig.data().transferStats)); - const sharedMetrics = new SharedStats( - new json_config.ChildConfig(statsConfig, statsConfig.data().hourlyMetrics), serverConfig, + const metricsConfig = readMetricsConfig(getPersistentFilename('shadowbox_stats.json')); + const managerMetrics = new ManagerMetrics( + new json_config.ChildConfig(metricsConfig, metricsConfig.data().transferStats)); + const sharedMetrics = new SharedMetrics( + new json_config.ChildConfig(metricsConfig, metricsConfig.data().hourlyMetrics), serverConfig, metricsUrl, new ip_location.MmdbLocationService()); logging.info('Starting...'); diff --git a/src/shadowbox/server/manager_metrics.spec.ts b/src/shadowbox/server/manager_metrics.spec.ts index 1df1b002..0d7611b1 100644 --- a/src/shadowbox/server/manager_metrics.spec.ts +++ b/src/shadowbox/server/manager_metrics.spec.ts @@ -14,7 +14,7 @@ import {InMemoryConfig} from '../infrastructure/json_config'; -import {ManagerStats, ManagerStatsJson} from './manager_metrics'; +import {ManagerMetrics, ManagerMetricsJson} from './manager_metrics'; function addDays(baseDate: Date, days: number) { const date = new Date(baseDate); @@ -22,26 +22,26 @@ function addDays(baseDate: Date, days: number) { return date; } -describe('ManagerStats', () => { +describe('ManagerMetrics', () => { it('Saves traffic to config', (done) => { const now = new Date(); - const config = new InMemoryConfig({} as ManagerStatsJson); - const stats = new ManagerStats(config); + const config = new InMemoryConfig({} as ManagerMetricsJson); + const metrics = new ManagerMetrics(config); - let report = stats.get30DayByteTransfer(); + let report = metrics.get30DayByteTransfer(); expect(report.bytesTransferredByUserId).toEqual({}); for (let di = 0; di < 40; di++) { - stats.recordBytesTransferred(addDays(now, -di), 'user-0', 1); + metrics.recordBytesTransferred(addDays(now, -di), 'user-0', 1); } - report = stats.get30DayByteTransfer(); + report = metrics.get30DayByteTransfer(); // This is being dropped expect(report.bytesTransferredByUserId).toEqual({'user-0': 30}); // We are not cleaning this from the config. expect(config.written.userIdSet).toEqual(['user-0']); expect(Object.keys(config.written.dailyUserBytesTransferred).length).toEqual(40); - expect(new ManagerStats(new InMemoryConfig(config.written)).get30DayByteTransfer()) + expect(new ManagerMetrics(new InMemoryConfig(config.written)).get30DayByteTransfer()) .toEqual(report); done(); }); diff --git a/src/shadowbox/server/manager_metrics.ts b/src/shadowbox/server/manager_metrics.ts index 9ee56f7f..7deec0da 100644 --- a/src/shadowbox/server/manager_metrics.ts +++ b/src/shadowbox/server/manager_metrics.ts @@ -16,22 +16,24 @@ import {JsonConfig} from '../infrastructure/json_config'; import {AccessKeyId} from '../model/access_key'; import {DataUsageByUser} from '../model/metrics'; -export interface ManagerStatsJson { +// Serialized format for the manager metrics. +// WARNING: Renaming fields will break backwards-compatibility. +export interface ManagerMetricsJson { // Bytes per user per day. The key encodes the user+day in the form "userId-dateInYYYYMMDD". dailyUserBytesTransferred?: Array<[string, number]>; - // Set of all User IDs for whom we have transfer stats. + // Set of all User IDs for whom we have transfer metrics. // TODO: Delete userIdSet. It can be derived from dailyUserBytesTransferred. userIdSet?: string[]; } -// ManagerStats keeps track of the number of bytes transferred per user, per day. +// ManagerMetrics keeps track of the number of bytes transferred per user, per day. // Surfaced by the manager service to display on the Manager UI. // TODO: Remove entries older than 30d. -export class ManagerStats { +export class ManagerMetrics { private dailyUserBytesTransferred: Map; private userIdSet: Set; - constructor(private config: JsonConfig) { + constructor(private config: JsonConfig) { const serializedObject = config.data(); if (serializedObject) { this.dailyUserBytesTransferred = new Map(serializedObject.dailyUserBytesTransferred); @@ -73,7 +75,7 @@ export class ManagerStats { // Returns the state of this object, e.g. // {"dailyUserBytesTransferred":[["0-20170816",100],["1-20170816",100]],"userIdSet":["0","1"]} - private toJson(target: ManagerStatsJson) { + private toJson(target: ManagerMetricsJson) { // Use [...] operator to serialize Map and Set objects to JSON. target.dailyUserBytesTransferred = [...this.dailyUserBytesTransferred]; target.userIdSet = [...this.userIdSet]; diff --git a/src/shadowbox/server/manager_service.ts b/src/shadowbox/server/manager_service.ts index 35147c48..93f9f2a9 100644 --- a/src/shadowbox/server/manager_service.ts +++ b/src/shadowbox/server/manager_service.ts @@ -19,7 +19,7 @@ import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; import {AccessKey, AccessKeyRepository} from '../model/access_key'; -import {ManagerStats} from './manager_metrics'; +import {ManagerMetrics} from './manager_metrics'; import {ServerConfigJson} from './server_config'; // Creates a AccessKey response. @@ -84,7 +84,7 @@ export class ShadowsocksManagerService { private defaultServerName: string, private serverConfig: JsonConfig, private accessKeys: AccessKeyRepository, - private managerMetrics: ManagerStats, + private managerMetrics: ManagerMetrics, ) {} public renameServer(req: RequestType, res: ResponseType, next: restify.Next): void { diff --git a/src/shadowbox/server/mocks/mocks.ts b/src/shadowbox/server/mocks/mocks.ts index 2a139ae8..ae253d2a 100644 --- a/src/shadowbox/server/mocks/mocks.ts +++ b/src/shadowbox/server/mocks/mocks.ts @@ -71,7 +71,7 @@ class MockShadowsocksInstance implements ShadowsocksInstance { export class MockShadowsocksServer { startInstance( - portNumber: number, password: string, statsSocket: dgram.Socket, + portNumber: number, password: string, metricsSocket: dgram.Socket, encryptionMethod?: string): Promise { const mock = new MockShadowsocksInstance(portNumber, password, encryptionMethod); return Promise.resolve(mock); diff --git a/src/shadowbox/server/server_access_key.ts b/src/shadowbox/server/server_access_key.ts index bc6930d9..e4ae1618 100644 --- a/src/shadowbox/server/server_access_key.ts +++ b/src/shadowbox/server/server_access_key.ts @@ -22,8 +22,8 @@ import {AccessKey, AccessKeyId, AccessKeyRepository} from '../model/access_key'; import {ShadowsocksInstance, ShadowsocksServer} from '../model/shadowsocks_server'; import {TextFile} from '../model/text_file'; -import {ManagerStats} from './manager_metrics'; -import {SharedStats} from './shared_metrics'; +import {ManagerMetrics} from './manager_metrics'; +import {SharedMetrics} from './shared_metrics'; // The format as json of access keys in the config file. interface AccessKeyConfig { @@ -90,16 +90,16 @@ class AccessKeyConfigFile { export function createServerAccessKeyRepository( proxyHostname: string, textFile: TextFile, shadowsocksServer: ShadowsocksServer, - managerMetrics: ManagerStats, sharedMetrics: SharedStats): Promise { + managerMetrics: ManagerMetrics, sharedMetrics: SharedMetrics): Promise { const configFile = new AccessKeyConfigFile(textFile); const configJson = configFile.loadConfig(); const reservedPorts = getReservedPorts(configJson.accessKeys); - // Create and save the stats socket. - return createBoundUdpSocket(reservedPorts).then((statsSocket) => { - reservedPorts.add(statsSocket.address().port); + // Create and save the metrics socket. + return createBoundUdpSocket(reservedPorts).then((metricsSocket) => { + reservedPorts.add(metricsSocket.address().port); return new ServerAccessKeyRepository( - proxyHostname, configFile, configJson, shadowsocksServer, statsSocket, managerMetrics, + proxyHostname, configFile, configJson, shadowsocksServer, metricsSocket, managerMetrics, sharedMetrics); }); } @@ -129,8 +129,8 @@ class ServerAccessKeyRepository implements AccessKeyRepository { constructor( private proxyHostname: string, private configFile: AccessKeyConfigFile, private configJson: ConfigJson, private shadowsocksServer: ShadowsocksServer, - private statsSocket: dgram.Socket, private managerMetrics: ManagerStats, - private sharedMetrics: SharedStats) { + private metricsSocket: dgram.Socket, private managerMetrics: ManagerMetrics, + private sharedMetrics: SharedMetrics) { for (const accessKeyJson of this.configJson.accessKeys) { this.startInstance(accessKeyJson).catch((error) => { logging.error(`Failed to start Shadowsocks instance for key ${accessKeyJson.id}: ${error}`); @@ -203,7 +203,7 @@ class ServerAccessKeyRepository implements AccessKeyRepository { private startInstance(accessKeyJson: AccessKeyConfig): Promise { return this.shadowsocksServer .startInstance( - accessKeyJson.port, accessKeyJson.password, this.statsSocket, + accessKeyJson.port, accessKeyJson.password, this.metricsSocket, accessKeyJson.encryptionMethod) .then((ssInstance) => { ssInstance.onInboundBytes( diff --git a/src/shadowbox/server/server_config.ts b/src/shadowbox/server/server_config.ts index f7b18a48..68d3dfb6 100644 --- a/src/shadowbox/server/server_config.ts +++ b/src/shadowbox/server/server_config.ts @@ -16,6 +16,8 @@ import * as uuidv4 from 'uuid/v4'; import * as json_config from '../infrastructure/json_config'; +// Serialized format for the server config. +// WARNING: Renaming fields will break backwards-compatibility. export interface ServerConfigJson { serverId: string; metricsEnabled: boolean; diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index d7dd2aba..a84e17a7 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -15,7 +15,7 @@ import * as https from 'https'; import * as ip_location from '../infrastructure/ip_location'; -import {PerUserStats} from '../model/metrics'; +import {PerUserMetrics} from '../model/metrics'; import * as shared_metrics from './shared_metrics'; @@ -185,7 +185,7 @@ describe('getHourlyServerMetricsReport', () => { }); }); -function getPerUserStats(ipAddresses: string[]): PerUserStats { +function getPerUserStats(ipAddresses: string[]): PerUserMetrics { return {bytesTransferred: 123, anonymizedIpAddresses: new Set(ipAddresses)}; } diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 3ec9c762..36d14911 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -19,13 +19,15 @@ import * as ip_location from '../infrastructure/ip_location'; import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; import {AccessKeyId} from '../model/access_key'; -import {PerUserStats} from '../model/metrics'; +import {PerUserMetrics} from '../model/metrics'; import {LastHourMetricsReadyCallback} from '../model/metrics'; import * as ip_util from './ip_util'; import {ServerConfigJson} from './server_config'; -export interface SharedStatsJson { +// Serialized format for the shared metrics. +// WARNING: Renaming fields will break backwards-compatibility. +export interface SharedMetricsJson { startTimestamp?: number; // TODO: Save the countries rather than anonymized IPs. There's no point in keeping the IPs. lastHourUserStatsObj?: @@ -34,31 +36,31 @@ export interface SharedStatsJson { const LAST_HOUR_METRICS_READY_EVENT = 'lastHourMetricsReady'; -// Keeps track of the connection stats per user, since the startDatetime. +// Keeps track of the connection metrics per user, since the startDatetime. // This is reported to the Outline team if the admin opts-in. -export class SharedStats { +export class SharedMetrics { private eventEmitter = new events.EventEmitter(); - // Date+time at which we started recording connection stats, e.g. + // Date+time at which we started recording connection metrics, e.g. // in case this object is constructed from data written to disk. public startDatetime: Date; - // Map from the metrics AccessKeyId to stats (bytes transferred, IP addresses). - public lastHourUserStats: Map; + // Map from the metrics AccessKeyId to metrics (bytes transferred, IP addresses). + public lastHourUserMetrics: Map; constructor( - private config: JsonConfig, + private config: JsonConfig, private serverConfig: JsonConfig, metricsUrl: string, ipLocationService: ip_location.IpLocationService) { const serializedObject = this.config.data(); this.startDatetime = serializedObject.startTimestamp ? new Date(serializedObject.startTimestamp) : new Date(); - this.lastHourUserStats = new Map(); + this.lastHourUserMetrics = new Map(); if (serializedObject.lastHourUserStatsObj) { Object.keys(serializedObject.lastHourUserStatsObj).map((userId) => { const perUserStatsObj = serializedObject.lastHourUserStatsObj[userId]; - this.lastHourUserStats.set(userId, { + this.lastHourUserMetrics.set(userId, { bytesTransferred: perUserStatsObj.bytesTransferred, anonymizedIpAddresses: new Set(perUserStatsObj.anonymizedIpAddresses) }); @@ -86,20 +88,20 @@ export class SharedStats { // CONSIDER: accepting hashedIpAddresses, which can be persisted to disk // and reported to the metrics server (to approximate number of devices per userId). recordBytesTransferred(userId: AccessKeyId, numBytes: number, ipAddresses: string[]) { - const perUserStats = this.lastHourUserStats.get(userId) || + const perUserMetrics = this.lastHourUserMetrics.get(userId) || {bytesTransferred: 0, anonymizedIpAddresses: new Set()}; - perUserStats.bytesTransferred += numBytes; + perUserMetrics.bytesTransferred += numBytes; const anonymizedIpAddresses = getAnonymizedAndDedupedIpAddresses(ipAddresses); for (const ip of anonymizedIpAddresses) { - perUserStats.anonymizedIpAddresses.add(ip); + perUserMetrics.anonymizedIpAddresses.add(ip); } - this.lastHourUserStats.set(userId, perUserStats); + this.lastHourUserMetrics.set(userId, perUserMetrics); this.toJson(this.config.data()); this.config.write(); } reset(): void { - this.lastHourUserStats = new Map(); + this.lastHourUserMetrics = new Map(); this.startDatetime = new Date(); this.toJson(this.config.data()); this.config.write(); @@ -117,11 +119,11 @@ export class SharedStats { // Returns the state of this object, e.g. // {"startTimestamp":1502896650353,"lastHourUserStatsObj":{"0":{"bytesTransferred":100,"anonymizedIpAddresses":["2620:0:1003:0:0:0:0:0","5.2.79.0"]}}} - private toJson(target: SharedStatsJson) { + private toJson(target: SharedMetricsJson) { // lastHourUserStats is a Map containing Set structures. Convert to an object // with array values. const lastHourUserStatsObj = {}; - this.lastHourUserStats.forEach((perUserStats, userId) => { + this.lastHourUserMetrics.forEach((perUserStats, userId) => { lastHourUserStatsObj[userId] = { bytesTransferred: perUserStats.bytesTransferred, anonymizedIpAddresses: [...perUserStats.anonymizedIpAddresses] @@ -133,17 +135,17 @@ export class SharedStats { } private generateHourlyReport(): void { - if (this.lastHourUserStats.size === 0) { - // No connection stats to report. + if (this.lastHourUserMetrics.size === 0) { + // No connection metrics to report. return; } this.eventEmitter.emit( LAST_HOUR_METRICS_READY_EVENT, this.startDatetime, new Date(), // endDatetime is the current date and time. - this.lastHourUserStats); + this.lastHourUserMetrics); - // Reset connection stats to begin recording the next hour. + // Reset connection metrics to begin recording the next hour. this.reset(); } } @@ -162,16 +164,16 @@ function getAnonymizedAndDedupedIpAddresses(ipAddresses: string[]): Set export function getHourlyServerMetricsReport( serverId: string, startDatetime: Date, endDatetime: Date, - lastHourUserStats: Map, + lastHourUserMetrics: Map, ipLocationService: ip_location.IpLocationService): Promise { - if (lastHourUserStats.size === 0) { - // Stats are empty, no need to post a report + if (lastHourUserMetrics.size === 0) { + // Metrics are empty, no need to post a report return Promise.resolve(null); } // convert lastHourUserStats to an array HourlyUserMetricsReport const userReportPromises = []; - lastHourUserStats.forEach((perUserStats, userId) => { - userReportPromises.push(getHourlyUserMetricsReport(userId, perUserStats, ipLocationService)); + lastHourUserMetrics.forEach((perUserMetrics, userId) => { + userReportPromises.push(getHourlyUserMetricsReport(userId, perUserMetrics, ipLocationService)); }); return Promise.all(userReportPromises).then((userReports: HourlyUserMetricsReport[]) => { // Remove any userReports containing sanctioned countries, and return @@ -222,10 +224,10 @@ interface HourlyUserMetricsReport { } function getHourlyUserMetricsReport( - userId: AccessKeyId, perUserStats: PerUserStats, + userId: AccessKeyId, perUserMetrics: PerUserMetrics, ipLocationService: ip_location.IpLocationService): Promise { const countryPromises = []; - for (const ip of perUserStats.anonymizedIpAddresses) { + for (const ip of perUserMetrics.anonymizedIpAddresses) { const countryPromise = ipLocationService.countryForIp(ip).catch((e) => { logging.warn(`Failed countryForIp call: ${e}`); return 'ERROR'; @@ -235,7 +237,7 @@ function getHourlyUserMetricsReport( return Promise.all(countryPromises).then((countries: string[]) => { return { userId, - bytesTransferred: perUserStats.bytesTransferred, + bytesTransferred: perUserMetrics.bytesTransferred, countries: getWithoutDuplicates(countries) }; });