diff --git a/jasmine.json b/jasmine.json index e5b81258..1f092dc0 100644 --- a/jasmine.json +++ b/jasmine.json @@ -1,6 +1,7 @@ { "spec_dir": ".", "spec_files": [ + "build/metrics_server/**/*.spec.js", "build/server_manager/electron_app/js/**/*.spec.js", "build/server_manager/web_app/**/*.spec.js", "build/shadowbox/js/**/*.spec.js" diff --git a/src/metrics_server/README.md b/src/metrics_server/README.md index 8f035f9d..45f03726 100644 --- a/src/metrics_server/README.md +++ b/src/metrics_server/README.md @@ -1,6 +1,43 @@ # Outline Metrics Server -The Outline Metrics Server is a [Google Cloud Function](https://cloud.google.com/functions/) which writes to BigQuery usage data received from Outline servers. +The Outline Metrics Server is a [Google App Engine](https://cloud.google.com/appengine) project that writes feature and connections metrics to BigQuery, as reported by opted-in Outline servers. + +## API + +### Endpoints + +The metrics server deploys two services: `dev`, used for development testing and debugging; and `prod`, used for production metrics. The `dev` environment is deployed to `https://dev.metrics.getoutline.org`; the `prod` environment is deployed to `https://prod.metrics.getoutline.org`. Each environment posts metrics to its own BigQuery dataset (see `config_[dev|prod].json`). + +### URLs + +The metrics server supports two URL paths: + +* `POST /connections`: report server data usage broken down by user. + + ``` + { + serverId: string, + startUtcMs: number, + endUtcMs: number, + userReports: [{ + userId: string, + countries: string[], + bytesTransferred: number, + }] + } + ``` +* `POST /features`: report feature usage. + + ``` + { + serverId: string, + serverVersion: string, + timestampUtcMs: number, + dataLimit: { + enabled: boolean + } + } + ``` ## Requirements @@ -12,15 +49,23 @@ The Outline Metrics Server is a [Google Cloud Function](https://cloud.google.com yarn do metrics_server/build ``` +## Run + +Run a local development metrics server: + +```sh +yarn do metrics_server/run +``` + ## Deploy * Authenticate with `gcloud`: ```sh gcloud auth login ``` -* To deploy to test: +* To deploy to dev: ```sh - yarn do metrics_server/deploy_test + yarn do metrics_server/deploy_dev ``` * To deploy to prod: ```sh @@ -29,35 +74,11 @@ yarn do metrics_server/build ## Test -We can test the function locally with the [Cloud Functions Emulator](https://cloud.google.com/functions/docs/emulator). - -**Note: The emulator is not actively maintained is very temperamental!** - -Because the emulator explicitly requests Node.js 6.x (it refuses to even install on other versions), we have not added it to our `package.json`. If you use a Node version manager such as [NVM](https://github.com/creationix/nvm), it is easy to switch temporarily to Node.js 6.x: -```sh -nvm install 6 -yarn global add @google-cloud/functions-emulator -``` - -`yarn do metrics_server/test` builds and spins up a local server. It accepts one argument, which it forwards to the function, e.g.: -``` -export TIMESTAMP=$(date +%s%3N) -yarn do metrics_server/test '{"serverId":"12345","startUtcMs":'$TIMESTAMP',"endUtcMs":'$(($TIMESTAMP+1))',"userReports":[{"userId":"1","bytesTransferred":60,"countries":["US","NL"]},{"userId":"2","bytesTransferred":100,"countries":["UK"]}]}' -``` - -After running, you can view the inserted data in BigQuery, e.g.: -```sql -SELECT * FROM [uproxysite:uproxy_metrics_test.connections_v1] ORDER BY endTimestamp DESC LIMIT 10; -``` - -**Note: The emulator ignores the response code: if the function returns 400 or 500, the command will appear to run successfully!** - -Troubleshooting: -* If you run into errors like `could not load the default credentials`, try this command: +* Unit test ```sh - gcloud auth application-default login + yarn do metrics_server/test + ``` +* Integration test + ```sh + yarn do metrics_server/test_integration ``` -* Many errors can be "fixed" by clearing the emulator's config, e.g.: - * `functions clear` - * kill any running server, e.g. `pkill -f functions` - * clear `~/.config/configstore/@google-cloud` diff --git a/src/metrics_server/app_dev.yaml b/src/metrics_server/app_dev.yaml new file mode 100644 index 00000000..0b859bb4 --- /dev/null +++ b/src/metrics_server/app_dev.yaml @@ -0,0 +1,7 @@ +runtime: nodejs10 +service: dev +handlers: +- url: /.* + script: auto + secure: always + redirect_http_response_code: 307 diff --git a/src/metrics_server/app_prod.yaml b/src/metrics_server/app_prod.yaml new file mode 100644 index 00000000..b3298144 --- /dev/null +++ b/src/metrics_server/app_prod.yaml @@ -0,0 +1,7 @@ +runtime: nodejs10 +service: prod +handlers: +- url: /.* + script: auto + secure: always + redirect_http_response_code: 307 diff --git a/src/metrics_server/config_dev.json b/src/metrics_server/config_dev.json new file mode 100644 index 00000000..59013283 --- /dev/null +++ b/src/metrics_server/config_dev.json @@ -0,0 +1,5 @@ +{ + "datasetName": "uproxy_metrics_dev", + "connectionMetricsTableName": "connections_v1", + "featureMetricsTableName": "feature_metrics" +} diff --git a/src/metrics_server/config_prod.json b/src/metrics_server/config_prod.json index 7c7056ef..dd780179 100644 --- a/src/metrics_server/config_prod.json +++ b/src/metrics_server/config_prod.json @@ -1,4 +1,5 @@ { "datasetName": "uproxy_metrics", - "tableName": "connections_v1" + "connectionMetricsTableName": "connections_v1", + "featureMetricsTableName": "feature_metrics" } diff --git a/src/metrics_server/config_test.json b/src/metrics_server/config_test.json deleted file mode 100644 index 3071fd06..00000000 --- a/src/metrics_server/config_test.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "datasetName": "uproxy_metrics_test", - "tableName": "connections_v1" -} diff --git a/src/metrics_server/connection_metrics.spec.ts b/src/metrics_server/connection_metrics.spec.ts new file mode 100644 index 00000000..062ac60d --- /dev/null +++ b/src/metrics_server/connection_metrics.spec.ts @@ -0,0 +1,252 @@ +// Copyright 2020 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {ConnectionRow, isValidConnectionMetricsReport, postConnectionMetrics} from './connection_metrics'; +import {InsertableTable} from './infrastructure/table'; +import {HourlyConnectionMetricsReport} from './model'; + +class FakeConnectionsTable implements InsertableTable { + public rows: ConnectionRow[]|undefined; + + async insert(rows: ConnectionRow[]) { + this.rows = rows; + } +} + +describe('postConnectionMetrics', () => { + it('correctly inserts feature metrics rows', async () => { + const table = new FakeConnectionsTable(); + const userReports = [ + { + userId: 'uid0', + countries: ['US', 'UK'], + bytesTransferred: 123, + }, + { + userId: 'uid1', + countries: ['EC'], + bytesTransferred: 456, + } + ]; + const report = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports}; + await postConnectionMetrics(table, report); + const rows: ConnectionRow[] = [ + { + serverId: report.serverId, + startTimestamp: new Date(report.startUtcMs).toISOString(), + endTimestamp: new Date(report.endUtcMs).toISOString(), + userId: userReports[0].userId, + bytesTransferred: userReports[0].bytesTransferred, + countries: userReports[0].countries + }, + { + serverId: report.serverId, + startTimestamp: new Date(report.startUtcMs).toISOString(), + endTimestamp: new Date(report.endUtcMs).toISOString(), + userId: userReports[1].userId, + bytesTransferred: userReports[1].bytesTransferred, + countries: userReports[1].countries + } + ]; + expect(table.rows).toEqual(rows); + }); +}); + +describe('isValidConnectionMetricsReport', () => { + it('returns true for valid report', () => { + const userReports = [ + {userId: 'uid0', countries: ['US', 'UK'], bytesTransferred: 123}, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const report = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports}; + expect(isValidConnectionMetricsReport(report)).toBeTruthy(); + }); + it('returns false for missing report', () => { + expect(isValidConnectionMetricsReport(undefined)).toBeFalsy(); + }); + it('returns false for inconsistent timestamp values', () => { + const userReports = [ + {userId: 'uid0', countries: ['US', 'UK'], bytesTransferred: 123}, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const invalidReport = { + serverId: 'id', + startUtcMs: 999, // startUtcMs > endUtcMs + endUtcMs: 1, + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport)).toBeFalsy(); + }); + it('returns false for out-of-bounds transferred bytes', () => { + const userReports = [ + { + userId: 'uid0', + countries: ['US', 'UK'], + bytesTransferred: -123 // Should not be negative + }, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const invalidReport = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports}; + expect(isValidConnectionMetricsReport(invalidReport)).toBeFalsy(); + + const userReports2 = [ + {userId: 'uid0', countries: ['US', 'UK'], bytesTransferred: 123}, { + userId: 'uid1', + countries: ['EC'], + bytesTransferred: 2 * Math.pow(2, 40) // 2TB is above the server capacity + } + ]; + const invalidReport2 = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports: userReports2}; + expect(isValidConnectionMetricsReport(invalidReport2)).toBeFalsy(); + }); + it('returns false for missing report fields', () => { + const invalidReport = { + // Missing `userReports` + serverId: 'id', + startUtcMs: 1, + endUtcMs: 2, + }; + expect(isValidConnectionMetricsReport(invalidReport)).toBeFalsy(); + + const invalidReport2 = { + serverId: 'id', + startUtcMs: 1, + endUtcMs: 2, + userReports: [] // Should not be empty + }; + expect(isValidConnectionMetricsReport(invalidReport2)).toBeFalsy(); + + const userReports = [ + {userId: 'uid0', countries: ['US', 'UK'], bytesTransferred: 123}, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const invalidReport3 = { + // Missing `serverId` + startUtcMs: 1, + endUtcMs: 2, + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport3)).toBeFalsy(); + + const invalidReport4 = { + // Missing `startUtcMs` + serverId: 'id', + endUtcMs: 2, + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport4)).toBeFalsy(); + + const invalidReport5 = { + // Missing `endUtcMs` + serverId: 'id', + startUtcMs: 2, + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport5)).toBeFalsy(); + }); + it('returns false for missing user report fields', () => { + const userReports = [ + { + // Missing `userId` + countries: ['US', 'UK'], + bytesTransferred: 123 + }, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const invalidReport = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports}; + expect(isValidConnectionMetricsReport(invalidReport)).toBeFalsy(); + + const userReports2 = [{ + // Missing `countries` + userId: 'uid0', + bytesTransferred: 123 + }]; + const invalidReport2 = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports: userReports2}; + expect(isValidConnectionMetricsReport(invalidReport2)).toBeFalsy(); + + const userReports3 = [{ + // Missing `bytesTransferred` + userId: 'uid0', + countries: ['US', 'UK'], + }]; + const invalidReport3 = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports: userReports3}; + expect(isValidConnectionMetricsReport(invalidReport3)).toBeFalsy(); + }); + it('returns false for incorrect report field types', () => { + const invalidReport = { + serverId: 'id', + startUtcMs: 1, + endUtcMs: 2, + userReports: [1, 2, 3] // Should be `HourlyUserConnectionMetricsReport[]` + }; + expect(isValidConnectionMetricsReport(invalidReport)).toBeFalsy(); + + const userReports = [ + {userId: 'uid0', countries: ['US', 'UK'], bytesTransferred: 123}, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const invalidReport2 = { + serverId: 987, // Should be a string + startUtcMs: 1, + endUtcMs: 2, + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport2)).toBeFalsy(); + + const invalidReport3 = { + serverId: 'id', + startUtcMs: '100', // Should be a number + endUtcMs: 200, + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport3)).toBeFalsy(); + + const invalidReport4 = { + // Missing `startUtcMs` + serverId: 'id', + startUtcMs: 1, + endUtcMs: '200', // Should be a number + userReports + }; + expect(isValidConnectionMetricsReport(invalidReport4)).toBeFalsy(); + }); + it('returns false for incorrect user report field types ', () => { + const userReports = [ + { + userId: 1234, // Should be a string + countries: ['US', 'UK'], + bytesTransferred: 123 + }, + {userId: 'uid1', countries: ['EC'], bytesTransferred: 456} + ]; + const invalidReport = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports}; + expect(isValidConnectionMetricsReport(invalidReport)).toBeFalsy(); + + const userReports2 = [{ + userId: 'uid0', + countries: [1, 2, 3], // Should be string[] + bytesTransferred: 123 + }]; + const invalidReport2 = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports: userReports2}; + expect(isValidConnectionMetricsReport(invalidReport2)).toBeFalsy(); + + const userReports3 = [{ + userId: 'uid0', + countries: ['US', 'UK'], + bytesTransferred: '1234', // Should be a number + }]; + const invalidReport3 = {serverId: 'id', startUtcMs: 1, endUtcMs: 2, userReports: userReports3}; + expect(isValidConnectionMetricsReport(invalidReport3)).toBeFalsy(); + }); +}); diff --git a/src/metrics_server/connection_metrics.ts b/src/metrics_server/connection_metrics.ts new file mode 100644 index 00000000..a08fb249 --- /dev/null +++ b/src/metrics_server/connection_metrics.ts @@ -0,0 +1,122 @@ +// Copyright 2020 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {Table} from '@google-cloud/bigquery'; +import {InsertableTable} from './infrastructure/table'; +import {HourlyConnectionMetricsReport, HourlyUserConnectionMetricsReport} from './model'; + +export interface ConnectionRow { + serverId: string; + startTimestamp: string; // ISO formatted string. + endTimestamp: string; // ISO formatted string. + userId: string; + bytesTransferred: number; + countries: string[]; +} + +export class BigQueryConnectionsTable implements InsertableTable { + constructor(private bigqueryTable: Table) {} + + async insert(rows: ConnectionRow[]): Promise { + await this.bigqueryTable.insert(rows); + } +} + +export function postConnectionMetrics( + table: InsertableTable, report: HourlyConnectionMetricsReport) { + return table.insert(getConnectionRowsFromReport(report)); +} + +function getConnectionRowsFromReport(report: HourlyConnectionMetricsReport): ConnectionRow[] { + const startTimestampStr = new Date(report.startUtcMs).toISOString(); + const endTimestampStr = new Date(report.endUtcMs).toISOString(); + const rows = []; + for (const userReport of report.userReports) { + rows.push({ + serverId: report.serverId, + startTimestamp: startTimestampStr, + endTimestamp: endTimestampStr, + userId: userReport.userId, + bytesTransferred: userReport.bytesTransferred, + countries: userReport.countries + }); + } + return rows; +} + +// Returns true iff testObject contains a valid HourlyConnectionMetricsReport. +// tslint:disable-next-line:no-any +export function isValidConnectionMetricsReport(testObject: any): + testObject is HourlyConnectionMetricsReport { + if (!testObject) { + return false; + } + + // Check that all required fields are present. + const requiredConnectionMetricsFields = ['serverId', 'startUtcMs', 'endUtcMs', 'userReports']; + for (const fieldName of requiredConnectionMetricsFields) { + if (!testObject[fieldName]) { + return false; + } + } + + // Check that `serverId` is a string. + if (typeof testObject.serverId !== 'string') { + return false; + } + + // Check timestamp types and that startUtcMs is not after endUtcMs. + if (typeof testObject.startUtcMs !== 'number' || typeof testObject.endUtcMs !== 'number' || + testObject.startUtcMs >= testObject.endUtcMs) { + return false; + } + + // Check that userReports is an array of 1 or more item. + if (!(testObject.userReports.length >= 1)) { + return false; + } + + const requiredUserReportFields = ['userId', 'countries', 'bytesTransferred']; + const MIN_BYTES_TRANSFERRED = 0; + const MAX_BYTES_TRANSFERRED = 1 * Math.pow(2, 40); // 1 TB. + for (const userReport of testObject.userReports) { + // Test that each `userReport` contains the required fields. + for (const fieldName of requiredUserReportFields) { + if (!userReport[fieldName]) { + return false; + } + } + // Check that `userId` is a string. + if (typeof userReport.userId !== 'string') { + return false; + } + + // Check that `bytesTransferred` is a number between min and max transfer limits + if (typeof userReport.bytesTransferred !== 'number' || + userReport.bytesTransferred < MIN_BYTES_TRANSFERRED || + userReport.bytesTransferred > MAX_BYTES_TRANSFERRED) { + return false; + } + + // Check that `countries` are strings. + for (const country of userReport.countries) { + if (typeof country !== 'string') { + return false; + } + } + } + + // Request is a valid HourlyConnectionMetricsReport. + return true; +} diff --git a/src/metrics_server/deploy_test_action.sh b/src/metrics_server/deploy_dev_action.sh similarity index 65% rename from src/metrics_server/deploy_test_action.sh rename to src/metrics_server/deploy_dev_action.sh index 360c8115..99e705d5 100755 --- a/src/metrics_server/deploy_test_action.sh +++ b/src/metrics_server/deploy_dev_action.sh @@ -14,10 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +SRC_DIR="src/metrics_server" +BUILD_DIR="build/metrics_server" + +rm -rf $BUILD_DIR + yarn do metrics_server/build -cp src/metrics_server/config_test.json build/metrics_server/config.json +cp $SRC_DIR/app_dev.yaml $BUILD_DIR/app.yaml +cp $SRC_DIR/config_dev.json $BUILD_DIR/config.json +cp $SRC_DIR/package.json $BUILD_DIR/ -cp src/metrics_server/package.json build/metrics_server/ - -gcloud --project=uproxysite functions deploy reportHourlyConnectionMetricsTest --trigger-http --source=build/metrics_server --entry-point=reportHourlyConnectionMetrics +gcloud app deploy $SRC_DIR/dispatch.yaml $BUILD_DIR --project uproxysite --verbosity info --promote --stop-previous-version diff --git a/src/metrics_server/deploy_prod_action.sh b/src/metrics_server/deploy_prod_action.sh index 3db582b8..a2a23457 100755 --- a/src/metrics_server/deploy_prod_action.sh +++ b/src/metrics_server/deploy_prod_action.sh @@ -14,10 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +SRC_DIR="src/metrics_server" +BUILD_DIR="build/metrics_server" + +rm -rf $BUILD_DIR + yarn do metrics_server/build -cp src/metrics_server/config_prod.json build/metrics_server/config.json +cp $SRC_DIR/app_prod.yaml $BUILD_DIR/app.yaml +cp $SRC_DIR/config_prod.json $BUILD_DIR/config.json +cp $SRC_DIR/package.json $BUILD_DIR/ -cp src/metrics_server/package.json build/metrics_server/ - -gcloud --project=uproxysite functions deploy reportHourlyConnectionMetrics --trigger-http --source=build/metrics_server --entry-point=reportHourlyConnectionMetrics +gcloud app deploy $SRC_DIR/dispatch.yaml $BUILD_DIR --project uproxysite --verbosity info --no-promote --no-stop-previous-version diff --git a/src/metrics_server/dispatch.yaml b/src/metrics_server/dispatch.yaml new file mode 100644 index 00000000..8520b49e --- /dev/null +++ b/src/metrics_server/dispatch.yaml @@ -0,0 +1,5 @@ +dispatch: + - url: "prod.metrics.getoutline.org/*" + service: prod + - url: "dev.metrics.getoutline.org/*" + service: dev diff --git a/src/metrics_server/feature_metrics.spec.ts b/src/metrics_server/feature_metrics.spec.ts new file mode 100644 index 00000000..7970887f --- /dev/null +++ b/src/metrics_server/feature_metrics.spec.ts @@ -0,0 +1,145 @@ +// Copyright 2020 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {FeatureRow, isValidFeatureMetricsReport, postFeatureMetrics} from './feature_metrics'; +import {InsertableTable} from './infrastructure/table'; +import {DailyFeatureMetricsReport} from './model'; + +class FakeFeaturesTable implements InsertableTable { + public rows: FeatureRow[]|undefined; + + async insert(rows: FeatureRow[]) { + this.rows = rows; + } +} + +describe('postFeatureMetrics', () => { + it('correctly inserts feature metrics rows', async () => { + const table = new FakeFeaturesTable(); + const report: DailyFeatureMetricsReport = { + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: {enabled: false} + }; + await postFeatureMetrics(table, report); + const rows: FeatureRow[] = [{ + serverId: report.serverId, + serverVersion: report.serverVersion, + timestamp: new Date(report.timestampUtcMs).toISOString(), + dataLimit: report.dataLimit + }]; + expect(table.rows).toEqual(rows); + }); +}); + +describe('isValidFeatureMetricsReport', () => { + it('returns true for valid report', () => { + const report = { + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(report)).toBeTruthy(); + }); + it('returns false for missing report', () => { + expect(isValidFeatureMetricsReport(undefined)).toBeFalsy(); + }); + it('returns false for incorrect report field types', () => { + const invalidReport = { + serverId: 1234, // Should be a string + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(invalidReport)).toBeFalsy(); + + const invalidReport2 = { + serverId: 'id', + serverVersion: 1010, // Should be a string + timestampUtcMs: 123456, + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(invalidReport2)).toBeFalsy(); + + const invalidReport3 = { + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: '123', // Should be a number + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(invalidReport3)).toBeFalsy(); + + const invalidReport4 = { + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: 'enabled' // Should be `DailyDataLimitMetricsReport` + }; + expect(isValidFeatureMetricsReport(invalidReport4)).toBeFalsy(); + + const invalidReport5 = { + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: { + enabled: 'true' // Should be a boolean + } + }; + expect(isValidFeatureMetricsReport(invalidReport5)).toBeFalsy(); + }); + it('returns false for missing report fields', () => { + const invalidReport = { + // Missing `serverId` + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(invalidReport)).toBeFalsy(); + + const invalidReport2 = { + // Missing `serverVersion` + serverId: 'id', + timestampUtcMs: 123456, + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(invalidReport2)).toBeFalsy(); + + const invalidReport3 = { + // Missing `timestampUtcMs` + serverId: 'id', + serverVersion: '0.0.0', + dataLimit: {enabled: true} + }; + expect(isValidFeatureMetricsReport(invalidReport3)).toBeFalsy(); + + const invalidReport4 = { + // Missing `dataLimit` + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: 123456, + }; + expect(isValidFeatureMetricsReport(invalidReport4)).toBeFalsy(); + + const invalidReport5 = { + // Missing `dataLimit.enabled` + serverId: 'id', + serverVersion: '0.0.0', + timestampUtcMs: 123456, + dataLimit: {} + }; + expect(isValidFeatureMetricsReport(invalidReport5)).toBeFalsy(); + }); +}); diff --git a/src/metrics_server/feature_metrics.ts b/src/metrics_server/feature_metrics.ts new file mode 100644 index 00000000..484e3de9 --- /dev/null +++ b/src/metrics_server/feature_metrics.ts @@ -0,0 +1,75 @@ +// Copyright 2020 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import {Table} from '@google-cloud/bigquery'; + +import {InsertableTable} from './infrastructure/table'; +import {DailyDataLimitMetricsReport, DailyFeatureMetricsReport} from './model'; + +// Reflects the feature metrics BigQuery table schema. +export interface FeatureRow { + serverId: string; + serverVersion: string; + timestamp: string; // ISO formatted string + dataLimit: DailyDataLimitMetricsReport; +} + +export class BigQueryFeaturesTable implements InsertableTable { + constructor(private bigqueryTable: Table) {} + + async insert(rows: FeatureRow|FeatureRow[]): Promise { + await this.bigqueryTable.insert(rows); + } +} + +export async function postFeatureMetrics( + table: InsertableTable, report: DailyFeatureMetricsReport) { + const featureRow: FeatureRow = { + serverId: report.serverId, + serverVersion: report.serverVersion, + timestamp: new Date(report.timestampUtcMs).toISOString(), + dataLimit: report.dataLimit + }; + return table.insert([featureRow]); +} + +// Returns true iff `obj` contains a valid DailyFeatureMetricsReport. +// tslint:disable-next-line:no-any +export function isValidFeatureMetricsReport(obj: any): obj is DailyFeatureMetricsReport { + if (!obj) { + return false; + } + + // Check that all required fields are present. + const requiredFeatureMetricsReportFields = + ['serverId', 'serverVersion', 'timestampUtcMs', 'dataLimit']; + for (const fieldName of requiredFeatureMetricsReportFields) { + if (!obj[fieldName]) { + return false; + } + } + + // Validate the report types are what we expect. + if (typeof obj.serverId !== 'string' || typeof obj.serverVersion !== 'string' || + typeof obj.timestampUtcMs !== 'number') { + return false; + } + + // Validate individual feature records. + if (typeof obj.dataLimit.enabled !== 'boolean') { + return false; + } + + return true; +} diff --git a/src/metrics_server/index.ts b/src/metrics_server/index.ts index 1a8beb1b..126c6929 100644 --- a/src/metrics_server/index.ts +++ b/src/metrics_server/index.ts @@ -1,4 +1,4 @@ -// Copyright 2018 The Outline Authors +// Copyright 2020 The Outline Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,39 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. +import {BigQuery} from '@google-cloud/bigquery'; import * as express from 'express'; import * as fs from 'fs'; import * as path from 'path'; -import {HourlyServerMetricsReport, isValidServerReport, postServerReport} from './post_server_report'; -// Accepts hourly connection metrics and inserts them into BigQuery. -// Request body should contain an HourlyServerMetricsReport. -exports.reportHourlyConnectionMetrics = (req: express.Request, res: express.Response) => { - if (req.method !== 'POST') { - res.status(405).send('Method not allowed'); - return; - } - if (!isValidServerReport(req.body)) { - res.status(400).send('Invalid request'); - return; - } - - const serverReport: HourlyServerMetricsReport = { - serverId: req.body.serverId, - startUtcMs: req.body.startUtcMs, - endUtcMs: req.body.endUtcMs, - userReports: req.body.userReports - }; - postServerReport(config.datasetName, config.tableName, serverReport).then(() => { - res.status(200).send('OK'); - }).catch((err: Error) => { - res.status(500).send('Error: ' + err); - }); -}; +import * as connections from './connection_metrics'; +import * as features from './feature_metrics'; interface Config { datasetName: string; - tableName: string; + connectionMetricsTableName: string; + featureMetricsTableName: string; } function loadConfig(): Config { @@ -52,4 +31,49 @@ function loadConfig(): Config { return JSON.parse(configText); } +const PORT = Number(process.env.PORT) || 8080; const config = loadConfig(); + +const bigqueryDataset = new BigQuery({projectId: 'uproxysite'}).dataset(config.datasetName); +const connectionsTable = new connections.BigQueryConnectionsTable( + bigqueryDataset.table(config.connectionMetricsTableName)); +const featuresTable = + new features.BigQueryFeaturesTable(bigqueryDataset.table(config.featureMetricsTableName)); + +const app = express(); +// Parse the request body for content-type 'application/json'. +app.use(express.json()); + +// Accepts hourly connection metrics and inserts them into BigQuery. +// Request body should contain an HourlyServerMetricsReport. +app.post('/connections', async (req: express.Request, res: express.Response) => { + try { + if (!connections.isValidConnectionMetricsReport(req.body)) { + res.status(400).send('Invalid request'); + return; + } + await connections.postConnectionMetrics(connectionsTable, req.body); + res.status(200).send('OK'); + } catch (err) { + res.status(500).send(`Error: ${err}`); + } +}); + +// Accepts daily feature metrics and inserts them into BigQuery. +// Request body should contain a `DailyFeatureMetricsReport`. +app.post('/features', async (req: express.Request, res: express.Response) => { + try { + if (!features.isValidFeatureMetricsReport(req.body)) { + res.status(400).send('Invalid request'); + return; + } + await features.postFeatureMetrics(featuresTable, req.body); + res.status(200).send('OK'); + } catch (err) { + res.status(500).send(`Error: ${err}`); + } +}); + +app.listen(PORT, () => { + console.log(`Metrics server listening on port ${PORT}`); +}); diff --git a/src/metrics_server/infrastructure/table.ts b/src/metrics_server/infrastructure/table.ts new file mode 100644 index 00000000..90e97fda --- /dev/null +++ b/src/metrics_server/infrastructure/table.ts @@ -0,0 +1,18 @@ +// Copyright 2020 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generic table interface that supports row insertion. +export interface InsertableTable { + insert(rows: T[]): Promise; +} diff --git a/src/metrics_server/model.ts b/src/metrics_server/model.ts new file mode 100644 index 00000000..6f7bccd6 --- /dev/null +++ b/src/metrics_server/model.ts @@ -0,0 +1,39 @@ +// Copyright 2020 The Outline Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// NOTE: These interfaces are mirrored in in src/shadowbox/server/metrics.ts +// Find a way to share them between shadowbox and metrics_server. +export interface HourlyConnectionMetricsReport { + serverId: string; + startUtcMs: number; + endUtcMs: number; + userReports: HourlyUserConnectionMetricsReport[]; +} + +export interface HourlyUserConnectionMetricsReport { + userId: string; + countries: string[]; + bytesTransferred: number; +} + +export interface DailyFeatureMetricsReport { + serverId: string; + serverVersion: string; + timestampUtcMs: number; + dataLimit: DailyDataLimitMetricsReport; +} + +export interface DailyDataLimitMetricsReport { + enabled: boolean; +} diff --git a/src/metrics_server/package.json b/src/metrics_server/package.json index 99f7ec61..084abb1f 100644 --- a/src/metrics_server/package.json +++ b/src/metrics_server/package.json @@ -9,10 +9,14 @@ "@google-cloud/storage here only to help Typescript code using @google-cloud/bigquery compile" ], "dependencies": { - "@google-cloud/bigquery": "^2.0.3" + "@google-cloud/bigquery": "^2.0.3", + "express": "^4.17.1" }, "devDependencies": { - "@types/express": "^4.0.36", + "@types/express": "^4.17.2", "@google-cloud/storage": "^2.3.1" + }, + "scripts": { + "start": "node ./index.js" } -} \ No newline at end of file +} diff --git a/src/metrics_server/post_server_report.ts b/src/metrics_server/post_server_report.ts deleted file mode 100644 index f9dd625c..00000000 --- a/src/metrics_server/post_server_report.ts +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright 2018 The Outline Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import {BigQuery} from '@google-cloud/bigquery'; - -// TODO(dborkan): HourlyServerMetricsReport and HourlyUserMetricsReport are -// copied from src/shadowbox/server/metrics.ts - find a way to share these -// definitions between shadowbox and the metrics_server. -export interface HourlyServerMetricsReport { - serverId: string; - startUtcMs: number; - endUtcMs: number; - userReports: HourlyUserMetricsReport[]; -} -interface HourlyUserMetricsReport { - userId: string; - countries: string[]; - bytesTransferred: number; -} - -interface ConnectionRow { - serverId: string; - startTimestamp: string; // ISO formatted string. - endTimestamp: string; // ISO formatted string. - userId: string; - bytesTransferred: number; - countries: string[]; -} - -// Instantiates a client -const bigqueryProject = new BigQuery({ - projectId: 'uproxysite' -}); - -export function postServerReport(datasetName: string, tableName: string, serverReport: HourlyServerMetricsReport) { - const dataset = bigqueryProject.dataset(datasetName); - const table = dataset.table(tableName); - const rows = getConnectionRowsFromServerReport(serverReport); - return new Promise((fulfill, reject) => { - table.insert(rows, (err) => { - if (err) { - reject(err); - } else { - fulfill(); - } - }); - }); -} - -function getConnectionRowsFromServerReport(serverReport: HourlyServerMetricsReport): ConnectionRow[] { - const startTimestampStr = new Date(serverReport.startUtcMs).toISOString(); - const endTimestampStr = new Date(serverReport.endUtcMs).toISOString(); - const rows = []; - for (const userReport of serverReport.userReports) { - rows.push({ - serverId: serverReport.serverId, - startTimestamp: startTimestampStr, - endTimestamp: endTimestampStr, - userId: userReport.userId, - bytesTransferred: userReport.bytesTransferred, - countries: userReport.countries - }); - } - return rows; -} - -// Returns true iff testObject contains a valid HourlyServerMetricsReport. -// tslint:disable-next-line:no-any -export function isValidServerReport(testObject: any): boolean { - // Check that all required fields are present. - const requiredServerReportFields = ['serverId', 'startUtcMs', 'endUtcMs', 'userReports']; - for (const fieldName of requiredServerReportFields) { - if (!testObject[fieldName]) { - return false; - } - } - - // Check that startUtcMs is not after endUtcMs. - if (testObject.startUtcMs >= testObject.endUtcMs) { - return false; - } - - // Check that userReports is an array of 1 or more item. - if (!(testObject.userReports.length >= 1)) { - return false; - } - - const requiredUserReportFields = ['userId', 'countries', 'bytesTransferred']; - const MIN_BYTES_TRANSFERRED = 0; - const MAX_BYTES_TRANSFERRED = 1 * Math.pow(2, 40); // 1 TB. - for (const userReport of testObject.userReports) { - // Test that each userReport contains valid fields. - for (const fieldName of requiredUserReportFields) { - if (!userReport[fieldName]) { - return false; - } - } - // Check that bytesTransferred is between min and max transfer limits - if (userReport.bytesTransferred < MIN_BYTES_TRANSFERRED || - userReport.bytesTransferred > MAX_BYTES_TRANSFERRED) { - return false; - } - } - - // Request is a valid HourlyServerMetricsReport. - return true; -} diff --git a/src/metrics_server/run_action.sh b/src/metrics_server/run_action.sh new file mode 100755 index 00000000..598fdb8b --- /dev/null +++ b/src/metrics_server/run_action.sh @@ -0,0 +1,25 @@ +#!/bin/bash -eu +# +# Copyright 2020 The Outline Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SRC_DIR="src/metrics_server" +BUILD_DIR="build/metrics_server" + +yarn do metrics_server/build + +cp $SRC_DIR/config_dev.json $BUILD_DIR/config.json +cp $SRC_DIR/package.json $BUILD_DIR/ + +yarn node $BUILD_DIR/index.js diff --git a/src/metrics_server/test_action.sh b/src/metrics_server/test_action.sh index eaeede88..959a1b28 100755 --- a/src/metrics_server/test_action.sh +++ b/src/metrics_server/test_action.sh @@ -14,21 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -if (( $# <= 0 )); then - echo "No test data specified" - exit 1; -fi - yarn do metrics_server/build - -cp src/metrics_server/config_test.json build/metrics_server/config.json - -# Because of weird issues with --local-path, have "functions deploy" search in the current -# directory instead. -pushd build/metrics_server -functions deploy reportHourlyConnectionMetrics --trigger-http - -functions call reportHourlyConnectionMetrics --data=$1 - -# Because the emulator ignores the response code, always print the logs to highlight any errors. -functions logs read +jasmine --config=$ROOT_DIR/jasmine.json diff --git a/src/metrics_server/test_integration_action.sh b/src/metrics_server/test_integration_action.sh new file mode 100755 index 00000000..7a8f226a --- /dev/null +++ b/src/metrics_server/test_integration_action.sh @@ -0,0 +1,96 @@ +#!/bin/bash -eu +# +# Copyright 2020 The Outline Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Metrics server integration test. Posts metrics to the development environment and queries BigQuery +# to ensure the rows have been inserted to the features and connections tables. +BIGQUERY_PROJECT=uproxysite +BIGQUERY_DATASET=uproxy_metrics_dev +CONNECTIONS_TABLE=connections_v1 +FEATURES_TABLE=feature_metrics + +METRICS_URL=https://dev.metrics.getoutline.org +CONNECTIONS_PATH=connections +FEATURES_PATH=features + +TMPDIR="$(mktemp -d)" +CONNECTIONS_REQUEST="$TMPDIR/connections.json" +CONNECTIONS_RESPONSE="$TMPDIR/connections_res.json" +CONNECTIONS_EXPECTED_RESPONSE="$TMPDIR/connections_expected_res.json" +FEATURES_REQUEST="$TMPDIR/features_req.json" +FEATURES_RESPONSE="$TMPDIR/features_res.json" +FEATURES_EXPECTED_RESPONSE="$TMPDIR/features_expected_res.json" + +TIMESTAMP=$(date +%s%3N) +SERVER_ID=$(uuidgen) +SERVER_VERSION=$(uuidgen) +USER_ID1=$(uuidgen) +USER_ID2=$(uuidgen) +# BYTES_TRANSFERRED2 < BYTES_TRANSFERRED1 so we can order the records before comparing them. +BYTES_TRANSFERRED1=$((2 + RANDOM % 100)) +BYTES_TRANSFERRED2=$(($BYTES_TRANSFERRED1 - 1)) + +echo "Using tmp directory $TMPDIR" + +# Write the request data to temporary files. +cat << EOF > $CONNECTIONS_REQUEST +{ + "serverId": "$SERVER_ID", + "startUtcMs": $TIMESTAMP, + "endUtcMs": $(($TIMESTAMP+1)), + "userReports": [{ + "userId": "$USER_ID1", + "bytesTransferred": $BYTES_TRANSFERRED1, + "countries": ["US", "NL"] + }, { + "userId": "$USER_ID2", + "bytesTransferred": $BYTES_TRANSFERRED2, + "countries": ["UK"] + }] +} +EOF +cat << EOF > $FEATURES_REQUEST +{ + "serverId": "$SERVER_ID", + "serverVersion": "$SERVER_VERSION", + "timestampUtcMs": $TIMESTAMP, + "dataLimit": { + "enabled": false + } +} +EOF + +# Write the expected responses to temporary files. +# Ignore the ISO formatted timestamps to ease the comparison. +cat << EOF > $CONNECTIONS_EXPECTED_RESPONSE +[{"bytesTransferred":"$BYTES_TRANSFERRED1","countries":["US","NL"],"serverId":"$SERVER_ID","userId":"$USER_ID1"},{"bytesTransferred":"$BYTES_TRANSFERRED2","countries":["UK"],"serverId":"$SERVER_ID","userId":"$USER_ID2"}] +EOF +cat << EOF > $FEATURES_EXPECTED_RESPONSE +[{"dataLimit":{"enabled":"false"},"serverId":"$SERVER_ID","serverVersion":"$SERVER_VERSION"}] +EOF + +echo "Connections request:" +cat $CONNECTIONS_REQUEST +curl -X POST -H "Content-Type: application/json" -d @$CONNECTIONS_REQUEST $METRICS_URL/connections && echo +sleep 5 +bq --project_id $BIGQUERY_PROJECT --format json query --nouse_legacy_sql 'SELECT serverId, userId, bytesTransferred, countries FROM `'"$BIGQUERY_DATASET.$CONNECTIONS_TABLE"'` WHERE serverId = "'"$SERVER_ID"'" ORDER BY bytesTransferred DESC LIMIT 2' > $CONNECTIONS_RESPONSE +diff $CONNECTIONS_RESPONSE $CONNECTIONS_EXPECTED_RESPONSE + +echo "Features request:" +cat $FEATURES_REQUEST +curl -X POST -H "Content-Type: application/json" -d @$FEATURES_REQUEST $METRICS_URL/features && echo +sleep 5 +bq --project_id $BIGQUERY_PROJECT --format json query --nouse_legacy_sql 'SELECT serverId, serverVersion, dataLimit FROM `'"$BIGQUERY_DATASET.$FEATURES_TABLE"'` WHERE serverId = "'"$SERVER_ID"'" ORDER BY timestamp DESC LIMIT 1' > $FEATURES_RESPONSE +diff $FEATURES_RESPONSE $FEATURES_EXPECTED_RESPONSE diff --git a/src/server_manager/electron_app/run_action.sh b/src/server_manager/electron_app/run_action.sh index 2325597d..e89a93f0 100755 --- a/src/server_manager/electron_app/run_action.sh +++ b/src/server_manager/electron_app/run_action.sh @@ -20,6 +20,6 @@ readonly NODE_MODULES_BIN_DIR=$ROOT_DIR/src/server_manager/node_modules/.bin cd $BUILD_DIR/server_manager/electron_app/static OUTLINE_DEBUG=true \ -SB_METRICS_URL=https://metrics-test.uproxy.org \ +SB_METRICS_URL=https://dev.metrics.getoutline.org \ SENTRY_DSN=https://ee9db4eb185b471ca08c8eb5efbf61f1@sentry.io/214597 \ $NODE_MODULES_BIN_DIR/electron . diff --git a/src/shadowbox/docker/cmd.sh b/src/shadowbox/docker/cmd.sh index 1eddb829..bb80854b 100755 --- a/src/shadowbox/docker/cmd.sh +++ b/src/shadowbox/docker/cmd.sh @@ -15,7 +15,7 @@ # limitations under the License. export SB_PUBLIC_IP=${SB_PUBLIC_IP:-$(curl https://ipinfo.io/ip)} -export SB_METRICS_URL=${SB_METRICS_URL:-https://metrics-prod.uproxy.org} +export SB_METRICS_URL=${SB_METRICS_URL:-https://prod.metrics.getoutline.org} # Make sure we don't leak readable files to other users. umask 0007 diff --git a/src/shadowbox/docker/run_action.sh b/src/shadowbox/docker/run_action.sh index 18c2607c..804481e1 100755 --- a/src/shadowbox/docker/run_action.sh +++ b/src/shadowbox/docker/run_action.sh @@ -34,6 +34,7 @@ declare -a docker_bindings=( -e SB_API_PREFIX=TestApiPrefix -e SB_CERTIFICATE_FILE=${SB_CERTIFICATE_FILE} -e SB_PRIVATE_KEY_FILE=${SB_PRIVATE_KEY_FILE} + -e SB_METRICS_URL=${SB_METRICS_URL} ) echo "Running image ${SB_IMAGE}" diff --git a/src/shadowbox/server/main.ts b/src/shadowbox/server/main.ts index 2a54df11..a86fdb4f 100644 --- a/src/shadowbox/server/main.ts +++ b/src/shadowbox/server/main.ts @@ -67,7 +67,7 @@ async function main() { // Default to production metrics, as some old Docker images may not have // SB_METRICS_URL properly set. - const metricsCollectorUrl = process.env.SB_METRICS_URL || 'https://metrics-prod.uproxy.org'; + const metricsCollectorUrl = process.env.SB_METRICS_URL || 'https://prod.metrics.getoutline.org'; if (!process.env.SB_METRICS_URL) { logging.warn('process.env.SB_METRICS_URL not set, using default'); } diff --git a/src/shadowbox/server/run_action.sh b/src/shadowbox/server/run_action.sh index cff7d463..e1602ea8 100755 --- a/src/shadowbox/server/run_action.sh +++ b/src/shadowbox/server/run_action.sh @@ -23,7 +23,7 @@ export LOG_LEVEL="${LOG_LEVEL:-debug}" export SB_PUBLIC_IP="${SB_PUBLIC_IP:-$(curl https://ipinfo.io/ip)}" # WARNING: The SB_API_PREFIX should be kept secret! export SB_API_PREFIX=TestApiPrefix -export SB_METRICS_URL=https://metrics-test.uproxy.org +export SB_METRICS_URL=https://dev.metrics.getoutline.org export SB_STATE_DIR=/tmp/outline source $ROOT_DIR/src/shadowbox/scripts/make_test_certificate.sh $SB_STATE_DIR diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index da954f91..fe62ef7b 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -15,9 +15,10 @@ import {ManualClock} from '../infrastructure/clock'; import {InMemoryConfig} from '../infrastructure/json_config'; import {AccessKeyId} from '../model/access_key'; +import {version} from '../package.json'; import {ServerConfigJson} from './server_config'; -import {HourlyServerMetricsReportJson, KeyUsage, MetricsCollectorClient, OutlineSharedMetricsPublisher, UsageMetrics} from './shared_metrics'; +import {DailyFeatureMetricsReportJson, HourlyServerMetricsReportJson, KeyUsage, MetricsCollectorClient, OutlineSharedMetricsPublisher, UsageMetrics} from './shared_metrics'; describe('OutlineSharedMetricsPublisher', () => { describe('Enable/Disable', () => { @@ -44,7 +45,7 @@ describe('OutlineSharedMetricsPublisher', () => { }); }); describe('Metrics Reporting', () => { - it('reports metrics correctly', async () => { + it('reports server usage metrics correctly', async () => { const clock = new ManualClock(); let startTime = clock.nowMs; const serverConfig = new InMemoryConfig({serverId: 'server-id'}); @@ -63,7 +64,7 @@ describe('OutlineSharedMetricsPublisher', () => { clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); - expect(metricsCollector.collectedReport).toEqual({ + expect(metricsCollector.collectedServerUsageReport).toEqual({ serverId: 'server-id', startUtcMs: startTime, endUtcMs: clock.nowMs, @@ -82,7 +83,7 @@ describe('OutlineSharedMetricsPublisher', () => { clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); - expect(metricsCollector.collectedReport).toEqual({ + expect(metricsCollector.collectedServerUsageReport).toEqual({ serverId: 'server-id', startUtcMs: startTime, endUtcMs: clock.nowMs, @@ -113,7 +114,7 @@ describe('OutlineSharedMetricsPublisher', () => { clock.nowMs += 60 * 60 * 1000; await clock.runCallbacks(); - expect(metricsCollector.collectedReport).toEqual({ + expect(metricsCollector.collectedServerUsageReport).toEqual({ serverId: 'server-id', startUtcMs: startTime, endUtcMs: clock.nowMs, @@ -125,14 +126,61 @@ describe('OutlineSharedMetricsPublisher', () => { publisher.stopSharing(); }); }); + it('reports feature metrics correctly', async () => { + const clock = new ManualClock(); + let timestamp = clock.nowMs; + const serverConfig = new InMemoryConfig( + {serverId: 'server-id', accessKeyDataLimit: {bytes: 123}}); + const metricsCollector = new FakeMetricsCollector(); + const publisher = new OutlineSharedMetricsPublisher( + clock, serverConfig, new ManualUsageMetrics(), (id: AccessKeyId) => '', metricsCollector); + + publisher.startSharing(); + await clock.runCallbacks(); + expect(metricsCollector.collectedFeatureMetricsReport).toEqual({ + serverId: 'server-id', + serverVersion: version, + timestampUtcMs: timestamp, + dataLimit: {enabled: true} + }); + clock.nowMs += 24 * 60 * 60 * 1000; + timestamp = clock.nowMs; + + delete serverConfig.data().accessKeyDataLimit; + await clock.runCallbacks(); + expect(metricsCollector.collectedFeatureMetricsReport).toEqual({ + serverId: 'server-id', + serverVersion: version, + timestampUtcMs: timestamp, + dataLimit: {enabled: false} + }); + }); + it('does not report metrics when sharing is disabled', async () => { + const clock = new ManualClock(); + const serverConfig = + new InMemoryConfig({serverId: 'server-id', metricsEnabled: false}); + const metricsCollector = new FakeMetricsCollector(); + spyOn(metricsCollector, 'collectServerUsageMetrics').and.callThrough(); + spyOn(metricsCollector, 'collectFeatureMetrics').and.callThrough(); + const publisher = new OutlineSharedMetricsPublisher( + clock, serverConfig, new ManualUsageMetrics(), (id: AccessKeyId) => '', metricsCollector); + + await clock.runCallbacks(); + expect(metricsCollector.collectServerUsageMetrics).not.toHaveBeenCalled(); + expect(metricsCollector.collectFeatureMetrics).not.toHaveBeenCalled(); + }); }); class FakeMetricsCollector implements MetricsCollectorClient { - public collectedReport: HourlyServerMetricsReportJson; + public collectedServerUsageReport: HourlyServerMetricsReportJson; + public collectedFeatureMetricsReport: DailyFeatureMetricsReportJson; - collectMetrics(report) { - this.collectedReport = report; - return Promise.resolve(); + async collectServerUsageMetrics(report) { + this.collectedServerUsageReport = report; + } + + async collectFeatureMetrics(report) { + this.collectedFeatureMetricsReport = report; } } diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 24659258..bf459fe8 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -19,10 +19,12 @@ import {JsonConfig} from '../infrastructure/json_config'; import * as logging from '../infrastructure/logging'; import {PrometheusClient} from '../infrastructure/prometheus_scraper'; import {AccessKeyId, AccessKeyMetricsId} from '../model/access_key'; +import {version} from '../package.json'; import {ServerConfigJson} from './server_config'; const MS_PER_HOUR = 60 * 60 * 1000; +const MS_PER_DAY = 24 * MS_PER_HOUR; const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); // Used internally to track key usage. @@ -49,6 +51,21 @@ export interface HourlyUserMetricsReportJson { bytesTransferred: number; } +// JSON format for the feature metrics report. +// Field renames will break backwards-compatibility. +export interface DailyFeatureMetricsReportJson { + serverId: string; + serverVersion: string; + timestampUtcMs: number; + dataLimit: DailyDataLimitMetricsReportJson; +} + +// JSON format for the data limit feature metrics report. +// Field renames will break backwards-compatibility. +export interface DailyDataLimitMetricsReportJson { + enabled: boolean; +} + export interface SharedMetricsPublisher { startSharing(); stopSharing(); @@ -92,18 +109,27 @@ export class PrometheusUsageMetrics implements UsageMetrics { } export interface MetricsCollectorClient { - collectMetrics(reportJson: HourlyServerMetricsReportJson): Promise; + collectServerUsageMetrics(reportJson: HourlyServerMetricsReportJson): Promise; + collectFeatureMetrics(reportJson: DailyFeatureMetricsReportJson): Promise; } export class RestMetricsCollectorClient { constructor(private serviceUrl: string) {} - collectMetrics(reportJson: HourlyServerMetricsReportJson): Promise { + collectServerUsageMetrics(reportJson: HourlyServerMetricsReportJson): Promise { + return this.postMetrics('/connections', JSON.stringify(reportJson)); + } + + collectFeatureMetrics(reportJson: DailyFeatureMetricsReportJson): Promise { + return this.postMetrics('/features', JSON.stringify(reportJson)); + } + + private postMetrics(urlPath: string, reportJson: string): Promise { const options = { - url: this.serviceUrl, + url: `${this.serviceUrl}${urlPath}`, headers: {'Content-Type': 'application/json'}, method: 'POST', - body: JSON.stringify(reportJson) + body: reportJson }; logging.info('Posting metrics: ' + JSON.stringify(options)); return new Promise((resolve, reject) => { @@ -142,10 +168,25 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { if (!this.isSharingEnabled()) { return; } - this.reportMetrics(await usageMetrics.getUsage()); - usageMetrics.reset(); + try { + await this.reportServerUsageMetrics(await usageMetrics.getUsage()); + usageMetrics.reset(); + } catch (err) { + console.error(`Failed to report server usage metrics: ${err}`); + } }, MS_PER_HOUR); // TODO(fortuna): also trigger report on shutdown, so data loss is minimized. + + this.clock.setInterval(async () => { + if (!this.isSharingEnabled()) { + return; + } + try { + this.reportFeatureMetrics(); + } catch (err) { + console.error(`Failed to report feature metrics: ${err}`); + } + }, MS_PER_DAY); } startSharing() { @@ -162,7 +203,7 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { return this.serverConfig.data().metricsEnabled || false; } - private async reportMetrics(usageMetrics: KeyUsage[]): Promise { + private async reportServerUsageMetrics(usageMetrics: KeyUsage[]): Promise { const reportEndTimestampMs = this.clock.now(); const userReports = [] as HourlyUserMetricsReportJson[]; @@ -190,7 +231,17 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { if (userReports.length === 0) { return; } - await this.metricsCollector.collectMetrics(report); + await this.metricsCollector.collectServerUsageMetrics(report); + } + + private async reportFeatureMetrics(): Promise { + const featureMetricsReport = { + serverId: this.serverConfig.data().serverId, + serverVersion: version, + timestampUtcMs: this.clock.now(), + dataLimit: {enabled: !!this.serverConfig.data().accessKeyDataLimit}, + }; + await this.metricsCollector.collectFeatureMetrics(featureMetricsReport); } }