From 1e6b6966c8f33cdd3c578b72a1d0bd51a32cd537 Mon Sep 17 00:00:00 2001 From: hassannagarajs2 Date: Wed, 1 Apr 2026 11:29:43 -0400 Subject: [PATCH 1/2] Remove staging and production deployment stages Removed staging and production deployment stages from the Jenkins pipeline. --- jenkinsfile.ecs | 126 +----------------------------------------------- 1 file changed, 1 insertion(+), 125 deletions(-) diff --git a/jenkinsfile.ecs b/jenkinsfile.ecs index ea977391..f4ce975e 100644 --- a/jenkinsfile.ecs +++ b/jenkinsfile.ecs @@ -125,132 +125,8 @@ pipeline { } } } - - stage('Build Staging') { - steps { - failSafeBuild('hcmi-cms-staging-config',CMS_PACKAGE_TYPE) - failSafeBuild('hcmi-api-staging-config',API_PACKAGE_TYPE) - failSafeBuild('hcmi-ui-staging-config',UI_PACKAGE_TYPE) - } - } - stage("Get Admin Permission to proceed to Staging") { - options { - timeout(time: 1, unit: 'HOURS') - } - when { - environment name: 'BUILD_STEP_SUCCESS', value: 'yes' - expression { - return env.BRANCH_NAME == 'master' || ( tag != '' && env.BRANCH_NAME == tag); - } - expression { - return tag != ''; - } - } - steps { - script { - env.DEPLOY_TO_STAGING = input message: 'User input required', - submitter: APP_ADMINS, - parameters: [choice(name: 'HCMI Portal: Deploy to STAGING Environment', choices: 'no\nyes', description: 'Choose "yes" if you want to deploy the STAGING server')] - } - } - } - stage('Deploy Staging') { - when { - environment name: 'BUILD_STEP_SUCCESS', value: 'yes' - environment name: 'DEPLOY_TO_STAGING', value: 'yes' - expression { - return env.BRANCH_NAME == 'master' || ( tag != '' && env.BRANCH_NAME == tag); - } - expression { - return tag != ''; - } - } - steps { - echo "DEPLOYING TO STAGING: (${env.BUILD_URL})" - sshagent (credentials: ["$STAGING_CREDS"]) { - sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$CMS_STAGING_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${CMS_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$CMS_STAGING_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$CMS_STAGING_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh staging $BUILD_NUMBER $CMS_PACKAGE_TYPE\"" - ) - sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$API_STAGING_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${API_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$API_STAGING_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$API_STAGING_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh staging $BUILD_NUMBER $API_PACKAGE_TYPE\"" - ) - sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$UI_STAGING_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${UI_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$UI_STAGING_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$UI_STAGING_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh staging $BUILD_NUMBER $UI_PACKAGE_TYPE\"" - ) - } - - echo "DEPLOYED TO STAGING: (${env.BUILD_URL})" - script { - env.STAGING_DEPLOYMENT_STATUS = 'SUCCESS' - } - - } - post { - failure { - echo "Deploy Failed: Branch '${env.BRANCH_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" - } - } - } - stage('Build PRD') { - steps { - failSafeBuild('hcmi-cms-prd-config',CMS_PACKAGE_TYPE) - failSafeBuild('hcmi-api-prd-config',API_PACKAGE_TYPE) - failSafeBuild('hcmi-ui-prd-config',UI_PACKAGE_TYPE) - } - } - stage("Get Admin Permission to proceed to PRD") { - options { - timeout(time: 1, unit: 'HOURS') - } - when { - environment name: 'BUILD_STEP_SUCCESS', value: 'yes' - expression { - return env.BRANCH_NAME == 'master' || ( tag != '' && env.BRANCH_NAME == tag); - } - expression { - return tag != ''; - } - } - steps { - script { - env.DEPLOY_TO_PRD = input message: 'User input required', - submitter: APP_ADMINS, - parameters: [choice(name: 'HCMI Portal: Deploy to PRD Environment', choices: 'no\nyes', description: 'Choose "yes" if you want to deploy the PRD server')] - } - } - } - stage('Deploy PRD') { - when { - environment name: 'BUILD_STEP_SUCCESS', value: 'yes' - environment name: 'DEPLOY_TO_PRD', value: 'yes' - expression { - return env.BRANCH_NAME == 'master' || ( tag != '' && env.BRANCH_NAME == tag); - } - expression { - return tag != ''; - } - } - steps { - echo "DEPLOYING TO PRD: (${env.BUILD_URL})" - sshagent (credentials: ["$PRD_CREDS"]) { - sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$CMS_PRD_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${CMS_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$CMS_PRD_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$CMS_PRD_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh prd $BUILD_NUMBER $CMS_PACKAGE_TYPE\"" - ) - sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$API_PRD_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${API_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$API_PRD_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$API_PRD_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh prd $BUILD_NUMBER $API_PACKAGE_TYPE\"" - ) - sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$UI_PRD_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${UI_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$UI_PRD_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$UI_PRD_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh prd $BUILD_NUMBER $UI_PACKAGE_TYPE\"" - ) - } - - echo "DEPLOYED TO PRD: (${env.BUILD_URL})" - script { - env.PRD_DEPLOYMENT_STATUS = 'SUCCESS' - } - - } - post { - failure { - echo "Deploy Failed: Branch '${env.BRANCH_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" - } - } - } } + post{ always { getPipelineResult() From c629ce0d10d311d75082b5d87c7e17b6d24a89d0 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 19 Aug 2026 10:26:31 -0400 Subject: [PATCH 2/2] #1142 OpenSearch & AWS migration (#1145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Testing Feb 6 * Import organize * Use Mongoose 8.23 * Updated searchClient names, imports, & usage -- draft * Fixed configureSets * mongoosastic, publish, and promise updates * Genes searchClient & async updates * Use module imports pt 1 * Revert "Use module imports pt 1" This reverts commit 99cfd466bcd95adc3b69ba3e52f2762cc18ca83c. * Replace use of logger.audit * Quick clean up * First Working Publish * Fixed Unpublish * Remove mongoosastic * Fix lastUpdated index setup issue * Fix cms client args * Fixed SelectAll Admin Table behaviour * File import and organization changes * Fixed bulk publish/unpublish toggle all & counts * Update searchClient definition * Fix duplicate indexing bug * Fix Somatic Variant Search * Genomic Variant falsy handling * Fixed Matched Models Publishing * MatchedModels UI improvements * Revised handling for Model document fields * Update lodash, remove elasticsearch, remove logging * Update lodash & mongoos * Fixed Matched Models Column * Use Arranger Server 3.0.3-beta.1, updated comments * Revert testing port change * Update docker-compose to use Opensearch * Use kebab case CMS folder name * Remove pnpm workspace, change function name casing * Docker compose edits * Sync data formatting * Remove extra logging * Fix republish import & add default index values * Additional comments for cleanMongoDoc * Fix UpdatedAt field * Fix Typo * Allow Nullable Fields * Current UpdateAt Model on Save * Use migrations config.js * Matched Models List value correction * Revert "Matched Models List value correction" This reverts commit f01707350354948daf25dbf807415c1d3ad37ac4. * Matched Models List bug * Remove logging * Minor doc updates * NODE_ENV in .schema, revert matched_models change * Additional matched models publish bug * Arranger metadata fixes * Additional searchClient config * Additional support for pm2 config * Handling for Mongo Collection in republish * Update express-restify-mongoose to fix 'delete' bug * Address axios & multer vulnerabilities * Updated Arranger Server * Use arranger-components 3.0.7 * New jenkinsfile * Update groovy comment * Updated URL regexes * Update Google Sheets URL handling * Google Template - Handle undefined Variant.find * Jenkinsfile Update * First Draft - Arranger router updates * Arranger rc-2 updates pt1 * Fixes to Arranger & Google Sheets * Use Arranger rc3 * Additional Support for pm2 * SearchClient pm2 Handling * lastUpdated changes * Testing - Server updates to support Auth changes * Update search client helper files * Update AWS Auth solution * Patch getClient pm2 usage * 🐛 Fix Bulk Variant Upload Sheet Creation * Hardens validation on Bulk Variant Upload Sheets by checking for and filtering out null Variant Names which were causing failures when adding validation data to the sheet templates * Use Arranger Router rc4, use separate pm2 file, update comments * CMS PM2 separate file * Add flag for GraphQL introspection * Updated yarn.lock * Fix Expanded column color contrast --------- Co-authored-by: Rakesh Mistry --- .gitignore | 5 +- README.md | 4 +- api/.env.schema | 6 +- api/index.js | 2 +- api/package.json | 11 +- api/src/dataExport.js | 13 +- api/src/health.js | 11 +- api/src/index.js | 53 - api/src/index.ts | 85 + api/src/lastUpdated.js | 14 +- api/src/pm2.ts | 21 + api/src/search.js | 16 +- api/src/services/elasticsearch.js | 7 - api/src/services/searchClient.ts | 35 + cms/env.schema | 1 + cms/package.json | 16 +- cms/src/helpers/dictionary.js | 43 +- cms/src/helpers/genomicVariants.js | 16 +- cms/src/helpers/matchedModels.js | 41 +- cms/src/helpers/uploadTemplate.js | 47 +- cms/src/helpers/validation.js | 20 +- cms/src/hooks.js | 15 +- cms/src/index.js | 52 +- cms/src/pm2.js | 14 + cms/src/routes/action.js | 10 +- cms/src/routes/bulk.js | 119 +- cms/src/routes/dictionary.js | 32 +- cms/src/routes/health.js | 6 +- cms/src/routes/publish.js | 22 +- cms/src/routes/sync-data.js | 106 +- cms/src/schemas/genes.js | 26 - cms/src/schemas/model.js | 240 +- .../services/elastic-search/common/client.js | 7 - .../elastic-search/common/schemas/model.js | 14 - cms/src/services/elastic-search/publish.js | 143 - cms/src/services/elastic-search/update.js | 23 - .../services/gdc-importer/VariantImporter.js | 97 +- cms/src/services/gdc-importer/mafFiles.js | 57 +- cms/src/services/publish/Publisher.js | 14 +- cms/src/services/s3/s3.js | 4 +- cms/src/services/search-client/client.js | 30 + .../genomicVariants.js | 80 +- .../search-client/indexLastUpdated.js | 26 + cms/src/services/search-client/indexModel.js | 27 + cms/src/services/search-client/publish.js | 307 ++ .../unpublish.js | 32 +- cms/src/validation/getPublishSchema.js | 15 +- .../{migrate-mongo-config.js => config.js} | 0 data_model/bin/fake.js | 2 +- data_model/package.json | 3 +- docker-compose.yml | 19 +- docker/elasticsearch/config/elasticsearch.yml | 10 - docker/elasticsearch/config/log4j2.properties | 9 - elasticsearch/arranger_metadata/base.json | 2 +- elasticsearch/arranger_metadata/extended.json | 497 +- elasticsearch/arranger_metadata/facets.json | 221 +- elasticsearch/arranger_metadata/matchbox.json | 10 +- elasticsearch/arranger_metadata/table.json | 421 +- elasticsearch/lastUpdated.json | 10 + jenkinsfile.ecs01 | 138 + package.json | 5 +- scripts/initializeEs.js | 18 +- scripts/initializeExpanded.js | 4 +- scripts/utils/esUtils.js | 57 +- scripts/utils/republishUtils.js | 25 +- ui/package.json | 2 +- ui/src/components/Model.jsx | 2 +- ui/src/components/TableMatchedModelsCell.jsx | 9 +- ui/src/components/admin/AdminView.jsx | 10 +- .../admin/Model/ModelSingleController.jsx | 4 +- .../admin/Model/actions/GenomicVariants.js | 32 +- .../admin/ModelsManager/ModelColumns.jsx | 2 +- .../ModelsManager/ModelManagerController.jsx | 529 +- .../admin/ModelsManager/ModelsManager.jsx | 58 +- .../GenomicVariantImportNotifications.jsx | 19 +- .../Notifications/NotificationToaster.jsx | 16 +- .../Notifications/PublishNotifications.jsx | 24 +- .../admin/Notifications/PublishProgress.jsx | 18 +- .../components/admin/Notifications/index.js | 13 +- .../components/admin/helpers/bulkActions.js | 1 - .../admin/helpers/fetchTableData.js | 12 +- .../components/admin/helpers/googleSheets.js | 26 +- .../components/admin/helpers/singleActions.js | 1 - .../components/admin/helpers/tableActions.js | 33 +- ui/src/components/admin/services/Fetcher.jsx | 2 +- ui/src/components/charts/TopVariantsChart.jsx | 298 +- ui/src/components/search/Search.jsx | 7 +- ui/src/theme/index.js | 4 +- ui/vite.config.ts | 2 +- yarn.lock | 4890 ++++++++--------- 90 files changed, 4875 insertions(+), 4575 deletions(-) delete mode 100644 api/src/index.js create mode 100644 api/src/index.ts create mode 100644 api/src/pm2.ts delete mode 100644 api/src/services/elasticsearch.js create mode 100644 api/src/services/searchClient.ts create mode 100644 cms/src/pm2.js delete mode 100644 cms/src/schemas/genes.js delete mode 100644 cms/src/services/elastic-search/common/client.js delete mode 100644 cms/src/services/elastic-search/common/schemas/model.js delete mode 100644 cms/src/services/elastic-search/publish.js delete mode 100644 cms/src/services/elastic-search/update.js create mode 100644 cms/src/services/search-client/client.js rename cms/src/services/{elastic-search => search-client}/genomicVariants.js (68%) create mode 100644 cms/src/services/search-client/indexLastUpdated.js create mode 100644 cms/src/services/search-client/indexModel.js create mode 100644 cms/src/services/search-client/publish.js rename cms/src/services/{elastic-search => search-client}/unpublish.js (55%) rename cms/variant-migrations/{migrate-mongo-config.js => config.js} (100%) delete mode 100644 docker/elasticsearch/config/elasticsearch.yml delete mode 100644 docker/elasticsearch/config/log4j2.properties create mode 100644 elasticsearch/lastUpdated.json create mode 100644 jenkinsfile.ecs01 diff --git a/.gitignore b/.gitignore index 9ab54d81..c09f2473 100644 --- a/.gitignore +++ b/.gitignore @@ -18,12 +18,9 @@ node_modules .env.production.local .vscode/ cms/env -cms/variant-migrations/config.js npm-debug.log* yarn-debug.log* yarn-error.log* -docker/mongo -docker/elasticsearch/data - +docker/ diff --git a/README.md b/README.md index e2011bcd..fc5628a7 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ To run the required migrations: ``` cd cms/variant-migrations -../../node_modules/.bin/migrate-mongo up -f migrate-mongo-config.js +../../node_modules/.bin/migrate-mongo up -f config.js ``` ### Quickstart @@ -59,7 +59,7 @@ yarn ``` cd cms/variant-migrations -../../node_modules/.bin/migrate-mongo up -f migrate-mongo-config.js +../../node_modules/.bin/migrate-mongo up -f config.js ``` 4. Initialize ElasticSearch: diff --git a/api/.env.schema b/api/.env.schema index 0b34cae8..cd55f49b 100644 --- a/api/.env.schema +++ b/api/.env.schema @@ -1,9 +1,11 @@ ES_URL= +ES_USER= +ES_PASS= ES_HOST= PROJECT_ID= PORT= ES_UPDATE_INDEX= ENABLE_ADMIN= -MONGODB_URI=mongodb://user@pass:localhost:27017/hcmi - LOG_LEVEL= +NODE_ENV= +SEARCH_CLIENT_TYPE= \ No newline at end of file diff --git a/api/index.js b/api/index.js index 52a38d85..110f1d5d 100644 --- a/api/index.js +++ b/api/index.js @@ -8,4 +8,4 @@ require('@babel/register')({ ], }); -require('./src/index'); +require('./src/index.ts'); diff --git a/api/package.json b/api/package.json index 3fde1c85..95c9cf6d 100644 --- a/api/package.json +++ b/api/package.json @@ -1,7 +1,7 @@ { "name": "@hcmi-portal/api", "version": "0.0.1", - "description": "``` npm i @overture-stack/arranger-server ```", + "description": "``` npm i @overture-stack/arranger-graphql-router ```", "private": true, "main": "index.js", "scripts": { @@ -11,8 +11,11 @@ "author": "", "license": "ISC", "dependencies": { - "@elastic/elasticsearch": "~7.17.0", - "@overture-stack/arranger-server": "^3.0.0", + "@aws-sdk/credential-provider-node": "^3.972.66", + "@opensearch-project/opensearch": "^3.6.0", + "@overture-stack/arranger-graphql-router": "1.0.0-rc.4", + "@overture-stack/arranger-types": "1.0.0-rc.3", + "@overture-stack/sqon-builder": "^1.1.0", "JSONStream": "^1.3.5", "babel-polyfill": "^6.26.0", "dotenv": "^5.0.1", @@ -21,7 +24,7 @@ "helmet": "^3.15.0", "jszip": "^3.7.0", "map-stream": "^0.0.7", - "mongoose": "7.8.7", + "mongoose": "^8.23.0", "pino": "^9.13.1", "pino-http": "^10.5.0", "sharp": "0.34.4" diff --git a/api/src/dataExport.js b/api/src/dataExport.js index a339daa4..3f5848ca 100644 --- a/api/src/dataExport.js +++ b/api/src/dataExport.js @@ -1,12 +1,13 @@ import express from 'express'; import bodyParser from 'body-parser'; import expressSanitizer from 'express-sanitizer'; -import { dataStream } from '@overture-stack/arranger-server/dist/download'; -import getAllData from '@overture-stack/arranger-server/dist/utils/getAllData'; +import { dataStream } from '@overture-stack/arranger-graphql-router/download'; +import { getAllData } from '@overture-stack/arranger-graphql-router/utils'; import JSZip from 'jszip'; import map from 'map-stream'; import through2 from 'through2'; -import getLogger from './logger'; + +import getLogger from './logger.js'; const logger = getLogger('dataExport'); @@ -48,7 +49,11 @@ dataExportRouter.post('/models', async (req, res) => { * We can get a stream from arranger using the getAllData method and then can store the variant data for each model * for future processing. */ - const allDataStream = await getAllData({ sqon, maxRows: 100, mock: {}, ctx: req.context }); + const allDataStream = await getAllData({ + sqon, + maxRows: 100, + ctx: req.context, + }); const collectVariantData = through2.obj(function ({ hits }, enc, callback) { hits.forEach((model) => { if (model?.genomic_variants && model?.genomic_variants.length > 0) { diff --git a/api/src/health.js b/api/src/health.js index c0276a80..61ac9034 100644 --- a/api/src/health.js +++ b/api/src/health.js @@ -1,14 +1,10 @@ import express from 'express'; -import elasticsearch from '@elastic/elasticsearch'; - import _ from 'lodash'; -const startTime = Date.now(); - -const client = new elasticsearch.Client({ - node: process.env.ES_URL, -}); +import pm2 from './pm2.ts'; +import getClient from './services/searchClient.ts'; +const startTime = Date.now(); const healthRouter = express.Router(); healthRouter.get('/', async (req, res) => { @@ -18,6 +14,7 @@ healthRouter.get('/', async (req, res) => { healthRouter.get('/es', async (req, res) => { try { + const client = getClient(pm2); const status = await client.ping(); if (_.get(status, 'statusCode') === 200) { const response = _.omit(status, 'meta'); diff --git a/api/src/index.js b/api/src/index.js deleted file mode 100644 index 46a97509..00000000 --- a/api/src/index.js +++ /dev/null @@ -1,53 +0,0 @@ -import 'babel-polyfill'; -import express from 'express'; -import { Server } from 'http'; -import ArrangerServer from '@overture-stack/arranger-server'; -import cors from 'cors'; -import lastUpdatedRouter from './lastUpdated'; -import healthRouter from './health'; -import searchRouter from './search'; -import dataExportRouter from './dataExport'; -import helmet from 'helmet'; -import bodyParser from 'body-parser'; -import expressSanitizer from 'express-sanitizer'; -import * as path from 'path'; -import getLogger from './logger'; - -const logger = getLogger('root'); -const port = process.env.PORT || 5050; -const app = express(); -const http = Server(app); - -app.use(helmet()); -app.use(bodyParser.json()); -app.use(bodyParser.urlencoded({ extended: true })); -app.use(expressSanitizer()); // each route is responsible for sanitization -app.use(cors()); - -//swagger -app.use('/docs', (req, res) => { - res.sendFile(path.join(__dirname, '../redoc.html')); -}); -app.use('/swagger', (req, res) => { - res.sendFile(path.join(__dirname, '../swagger.json')); -}); - -const appConfig = { - configsSource: '../elasticsearch/arranger_metadata/', - enableAdmin: process.env.ENABLE_ADMIN || false, - enableLogs: process.env.ENABLE_LOGS || false, - esHost: process.env.ES_HOST, - graphqlOptions: {}, -}; - -ArrangerServer(appConfig).then((router) => { - app.use(router); - app.use('/last-updated', lastUpdatedRouter); - app.use('/health', healthRouter); - app.use('/search', searchRouter); - app.use('/export', dataExportRouter); - - http.listen(port, async () => { - logger.info({ port }, 'API Started!'); - }); -}); diff --git a/api/src/index.ts b/api/src/index.ts new file mode 100644 index 00000000..9ece816e --- /dev/null +++ b/api/src/index.ts @@ -0,0 +1,85 @@ +import 'babel-polyfill'; +import bodyParser from 'body-parser'; +import cors from 'cors'; +import express from 'express'; +import expressSanitizer from 'express-sanitizer'; +import helmet from 'helmet'; +import { Server } from 'http'; +import ArrangerRouter, { type ArrangerBaseContext } from '@overture-stack/arranger-graphql-router'; +import type { ConfigsObject, DisplayType, ExtendedConfigs, FacetsConfigs, MatchBoxConfigs, TableConfigs } from '@overture-stack/arranger-types/configs'; +import * as path from 'path'; + +import baseConfig from '../../elasticsearch/arranger_metadata/base.json' with { type: 'json' }; +import extendedConfigFile from '../../elasticsearch/arranger_metadata/extended.json' with { type: 'json' }; +import facetsConfigFile from '../../elasticsearch/arranger_metadata/facets.json' with { type: 'json' }; +import matchboxConfigFile from '../../elasticsearch/arranger_metadata/matchbox.json' with { type: 'json' }; +import tableConfigFile from '../../elasticsearch/arranger_metadata/table.json' with { type: 'json' }; + +import dataExportRouter from './dataExport.js'; +import getClient from './services/searchClient.ts'; +import getLogger from './logger.js'; +import healthRouter from './health.js'; +import lastUpdatedRouter from './lastUpdated.js'; +import pm2 from './pm2.ts'; +import searchRouter from './search.js'; + +const logger = getLogger('root'); +const port = process.env.PORT || 5050; +const app = express(); +const http = new Server(app); + +app.use(helmet()); +app.use(bodyParser.json()); +app.use(bodyParser.urlencoded({ extended: true })); +app.use(expressSanitizer()); // each route is responsible for sanitization +app.use(cors()); + +// Swagger +app.use('/docs', (req, res) => { + res.sendFile(path.join(__dirname, '../redoc.html')); +}); +app.use('/swagger', (req, res) => { + res.sendFile(path.join(__dirname, '../swagger.json')); +}); + +// Arranger Configs +const displayTypeValues: DisplayType[] = ['all' , 'bits' , 'boolean' , 'bytes' , 'date' , 'list' , 'nested' , 'number']; + +const extendedConfigs: ExtendedConfigs[] = extendedConfigFile['extended'].map(configRecord => { + const {type: stringType} = configRecord; + const displayType = displayTypeValues.find(type => stringType === type) || 'all'; + return {...configRecord, type: displayType}; +}); +const facetConfigs: FacetsConfigs = facetsConfigFile['facets']; +const matchboxConfigs: MatchBoxConfigs[] = matchboxConfigFile['matchbox']; +const tableConfigs: TableConfigs = tableConfigFile['table']; + +const disableGraphQLIntrospection = process.env.DISABLE_GRAPHQL_INTROSPECTION === 'true' || pm2.DISABLE_GRAPHQL_INTROSPECTION || false; +const esHost = process.env.ES_HOST || pm2.ES_URL || 'http://localhost:9200'; + +const configs: Partial> = { + ...baseConfig, + disableGraphQLIntrospection, + esHost, + esIndex: 'hcmi', + extended: extendedConfigs, + enableDebug: true, + facets: facetConfigs, + matchbox: matchboxConfigs, + maxDepth: 10, + table: tableConfigs +}; + +const esClient = getClient(pm2); + +ArrangerRouter({ configs, esClient }).then((router) => { + app.use(router); + app.use('/last-updated', lastUpdatedRouter); + app.use('/health', healthRouter); + app.use('/search', searchRouter); + app.use('/export', dataExportRouter); + + http.listen(port, async () => { + logger.info({ port }, 'API Started!'); + }); +}); diff --git a/api/src/lastUpdated.js b/api/src/lastUpdated.js index 78cdafd6..94e8891f 100644 --- a/api/src/lastUpdated.js +++ b/api/src/lastUpdated.js @@ -1,14 +1,19 @@ // @ts-check import express from 'express'; -import esClient from './services/elasticsearch'; + +import pm2 from './pm2.ts'; +import getClient from './services/searchClient.ts'; +import getLogger from './logger.js'; const lastUpdatedRouter = express.Router(); +const logger = getLogger('lastUpdated Router'); lastUpdatedRouter.get('/', async (req, res) => { + const searchClient = getClient(pm2); try { - const response = await esClient.search({ - index: process.env.ES_UPDATE_INDEX, + const response = await searchClient?.search({ + index: process.env.ES_UPDATE_INDEX || pm2.ES_UPDATE_INDEX || 'hcmi-update', body: { query: { match_all: {}, @@ -23,8 +28,9 @@ lastUpdatedRouter.get('/', async (req, res) => { ], }, }); - return res.json(response.body.hits.hits[0]._source); + return res.json(response?.body?.hits?.hits[0]?._source); } catch (error) { + logger.error(`Error retrieving last updated date from Arranger: ${error}`); return res.status(500).json({ error: error, }); diff --git a/api/src/pm2.ts b/api/src/pm2.ts new file mode 100644 index 00000000..cf4c143d --- /dev/null +++ b/api/src/pm2.ts @@ -0,0 +1,21 @@ +import pm2Config from './../pm2.config.js'; + +// PM2 Env Setup +type pm2EnvValues = 'dev' | 'prd' | 'staging'; + +const pm2EnvValues = ['dev', 'prd', 'staging']; +const pm2Env: pm2EnvValues = + process.env.ENV && pm2EnvValues.includes(process.env.ENV) + ? (process.env.ENV as pm2EnvValues) + : 'dev'; + +const pm2ConfigGeneric = + (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0].env) || {}; +const pm2ConfigForEnv = + (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0][`env_${pm2Env}`]) || {}; + +const pm2 = { ...pm2ConfigGeneric, ...pm2ConfigForEnv }; + +export type PM2Config = typeof pm2; + +export default pm2; diff --git a/api/src/search.js b/api/src/search.js index c143ec75..25138f4b 100644 --- a/api/src/search.js +++ b/api/src/search.js @@ -1,6 +1,8 @@ import express from 'express'; -import esClient from './services/elasticsearch'; -import { get } from 'lodash'; +import _ from 'lodash'; + +import pm2 from './pm2.ts'; +import getClient from './services/searchClient.ts'; const GENES_INDEX = 'genes'; const VARIANTS_INDEX = 'genomic_variants'; @@ -28,11 +30,12 @@ geneSearchRouter.get('/gene', async (req, res) => { }, }; - const response = await esClient.search({ + const searchClient = getClient(pm2); + const response = await searchClient.search({ index: GENES_INDEX, body: { query }, }); - const genes = get(response, 'body.hits.hits', []).map((i) => i._source); + const genes = _.get(response, 'body.hits.hits', []).map((i) => i._source); res.status(200).json({ genes }); } catch (err) { console.log('Failure performing gene search:', err); @@ -57,11 +60,12 @@ geneSearchRouter.get('/variant', async (req, res) => { }, }; - const response = await esClient.search({ + const searchClient = getClient(pm2); + const response = await searchClient.search({ index: VARIANTS_INDEX, body: { query }, }); - const genes = get(response, 'body.hits.hits', []).map((i) => i._source); + const genes = _.get(response, 'body.hits.hits', []).map((i) => i._source); res.status(200).json({ genes }); } catch (err) { console.log('Failure performing genomic variants search:', err); diff --git a/api/src/services/elasticsearch.js b/api/src/services/elasticsearch.js deleted file mode 100644 index 1f8d63dc..00000000 --- a/api/src/services/elasticsearch.js +++ /dev/null @@ -1,7 +0,0 @@ -import elasticsearch from '@elastic/elasticsearch'; - -const client = new elasticsearch.Client({ - node: process.env.ES_URL, -}); - -export default client; diff --git a/api/src/services/searchClient.ts b/api/src/services/searchClient.ts new file mode 100644 index 00000000..dae4e90d --- /dev/null +++ b/api/src/services/searchClient.ts @@ -0,0 +1,35 @@ +import { Client } from '@opensearch-project/opensearch'; +import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; + +import type { PM2Config } from './../pm2.ts'; + +const getClient = (pm2Configs: PM2Config | undefined) => { + const node = process.env.ES_URL || pm2Configs?.ES_URL || 'http://localhost:9200'; + const authType = process.env.SEARCH_ENGINE_AUTH_TYPE || pm2Configs?.SEARCH_ENGINE_AUTH_TYPE || ''; + const region = + process.env.SEARCH_ENGINE_AUTH_REGION || pm2Configs?.SEARCH_ENGINE_AUTH_REGION || ''; + const service = + process.env.SEARCH_ENGINE_AUTH_SERVICE || pm2Configs?.SEARCH_ENGINE_AUTH_SERVICE || ''; + const username = process.env.ES_USER || pm2Configs?.ES_USER || ''; + const password = process.env.ES_PASS || pm2Configs?.ES_PASS || ''; + + const esClient = + authType === 'aws' + ? new Client({ + ...AwsSigv4Signer({ + region, + service: service as 'es' | 'aoss', + getCredentials: () => defaultProvider()(), + }), + node, + }) + : new Client({ + node, + auth: username ? { username, password } : undefined, + }); + + return esClient; +}; + +export default getClient; diff --git a/cms/env.schema b/cms/env.schema index 9a9526aa..5c17ba26 100644 --- a/cms/env.schema +++ b/cms/env.schema @@ -14,5 +14,6 @@ ES_PORT= ES_API_VERSION= ES_LOG_LEVEL= AUTH_ENABLED= +SEARCH_CLIENT_TYPE= LOG_LEVEL= diff --git a/cms/package.json b/cms/package.json index 034e9663..fc1ba0a7 100644 --- a/cms/package.json +++ b/cms/package.json @@ -12,9 +12,12 @@ "author": "OICR", "license": "ISC", "dependencies": { - "@elastic/elasticsearch": "~7.17.0", + "@aws-sdk/credential-provider-node": "^3.972.66", + "@opensearch-project/opensearch": "^3.6.0", + "@overture-stack/arranger-graphql-router": "1.0.0-rc.4", + "@overture-stack/arranger-types": "1.0.0-rc.3", "aws-sdk": "^2.723.0", - "axios": "^1.7.9", + "axios": "^1.16.0", "babel-polyfill": "^6.26.0", "cors": "^2.8.4", "csvtojson": "^2.0.14", @@ -23,19 +26,18 @@ "dotenv": "^5.0.1", "elasticsearch": "^16.6.0", "express": "^4.16.3", - "express-restify-mongoose": "^7.0.2", + "express-restify-mongoose": "^9.0.10", "fs": "^0.0.1-security", "google-auth-library": "^8.4.0", "googleapis": "100.0.0", "helmet": "^3.15.0", "json2csv": "^4.2.1", - "lodash": "^4.17.21", + "lodash": "^4.18.1", "map-stream": "^0.0.7", "method-override": "^2.3.10", - "mongoose": "7.8.7", - "mongoose-elasticsearch-xp": "^5.8.0", + "mongoose": "^8.23.0", "morgan": "^1.10.1", - "multer": "^2.0.2", + "multer": "^2.1.1", "pino": "^9.13.1", "pino-http": "^10.5.0", "sharp": "0.34.4", diff --git a/cms/src/helpers/dictionary.js b/cms/src/helpers/dictionary.js index 3d57807c..7547ca73 100644 --- a/cms/src/helpers/dictionary.js +++ b/cms/src/helpers/dictionary.js @@ -35,21 +35,21 @@ export const getDictionaryOptions = async () => { const output = dictionary.reduce((acc, field) => { let { name, values } = field; - acc[`${name}Options`] = values.map(i => i.value); + acc[`${name}Options`] = values.map((i) => i.value); return acc; }, {}); - const ctd = dictionary.find(i => i.name === 'clinicalTumorDiagnosis'); + const ctd = dictionary.find((i) => i.name === 'clinicalTumorDiagnosis'); const ctdDependentOptions = {}; - ctd.dependentValues.forEach(val => (ctdDependentOptions[val] = {})); + ctd.dependentValues.forEach((val) => (ctdDependentOptions[val] = {})); - ctd.values.forEach(val => { + ctd.values.forEach((val) => { const valueName = val.value; - val.dependents.forEach(dependent => { + val.dependents.forEach((dependent) => { const dependentName = dependent.name; - const dependentValues = dependent.values.map(val => val.value); + const dependentValues = dependent.values.map((val) => val.value); ctdDependentOptions[dependentName][valueName.toLowerCase()] = dependentValues; }); @@ -63,7 +63,7 @@ export const resetDraft = async () => { const dictionary = await Dictionary.findOne({}, {}, { sort: { created_at: -1 } }); const draft = new Draft({ fields: dictionary.fields }); await draft.save(); - logger.audit({}, 'reset draft', 'Dictionary draft reset'); + logger.info({}, 'reset draft', 'Dictionary draft reset'); return await Draft.findOne({}, {}, { sort: { created_at: -1 } }); }; @@ -77,12 +77,12 @@ export const publishDraft = async () => { // find edited values const edits = []; - draft.fields.forEach(field => { + draft.fields.forEach((field) => { if (field.dependentValues.length > 0) { // dependent field case - field.values.forEach(value => { - value.dependents.forEach(dependent => { - dependent.values.forEach(dependentValue => { + field.values.forEach((value) => { + value.dependents.forEach((dependent) => { + dependent.values.forEach((dependentValue) => { if (dependentValue.status === draftStatus.edited) { edits.push({ field: field.name, @@ -97,7 +97,7 @@ export const publishDraft = async () => { }); } // basic field case - field.values.forEach(value => { + field.values.forEach((value) => { if (value.status === draftStatus.edited) { edits.push({ field: field.name, @@ -119,19 +119,22 @@ export const publishDraft = async () => { const dictionary = new Dictionary({ fields: draft.fields }); await dictionary.save(); - logger.audit({ edits }, 'dictionary created', 'Created new dictionary from draft values'); + logger.info({ edits }, 'dictionary created', 'Created new dictionary from draft values'); - models.forEach(model => { + models.forEach((model) => { // Find these values in the models. let edited = false; - edits.forEach(edit => { + edits.forEach((edit) => { const editField = edit.dependentName ? edit.dependentName : edit.field; // - update those models and change their status if (editField === 'therapy' && model.therapy.includes(edit.original)) { // special case, therapy is an array - model.therapy.splice(model.therapy.indexOf(val => val === edit.original), 1); + model.therapy.splice( + model.therapy.indexOf((val) => val === edit.original), + 1, + ); model.therapy.push(edit.value); model.status = model.status === modelStatus.published ? modelStatus.unpublishedChanges : model.status; @@ -148,7 +151,7 @@ export const publishDraft = async () => { }); if (edited) { model.save(); - logger.audit( + logger.info( { model: model.name }, 'model saved', 'Model values updated due to dictionary publish', @@ -160,7 +163,7 @@ export const publishDraft = async () => { return { dictionary, updatedModels }; }; -export const countDraftStats = draft => { +export const countDraftStats = (draft) => { const output = { edited: 0, new: 0 }; const hasDependencies = draft.dependentValues && draft.dependentValues.length > 0; @@ -202,12 +205,12 @@ export const editValue = (target, original, updated) => { }; export const valueExists = (valuesList, value) => { - return !!valuesList.find(val => val.value === value); + return !!valuesList.find((val) => val.value === value); }; export const removeValueIfNew = (valueList, value) => { const index = valueList.indexOf( - valueList.find(i => i.status === draftStatus.new && i.value === value), + valueList.find((i) => i.status === draftStatus.new && i.value === value), ); if (index >= 0) { valueList.splice(index, 1); diff --git a/cms/src/helpers/genomicVariants.js b/cms/src/helpers/genomicVariants.js index ce90756c..0f774df4 100644 --- a/cms/src/helpers/genomicVariants.js +++ b/cms/src/helpers/genomicVariants.js @@ -10,7 +10,7 @@ import { import getLogger from '../logger.js'; const logger = getLogger('helpers/genomicVariants'); -export const clearGenomicVariants = async name => { +export const clearGenomicVariants = async (name) => { // Stop any active imports, if any: try { await VariantImporter.stopImport(name); @@ -37,8 +37,8 @@ export const clearGenomicVariants = async name => { } }; -const titleCase = (text, filter = i => true) => { - return text.replace(/\w\S*/g, i => { +const titleCase = (text, filter = (i) => true) => { + return text.replace(/\w\S*/g, (i) => { if (filter(i)) { return i.charAt(0).toUpperCase() + i.substr(1); } @@ -68,9 +68,9 @@ const buildVariantId = ({ } }; -const buildModelUrl = caseId => `${BASE_GDC_URL}/cases/${caseId}`; -const buildMafUrl = fileId => `${BASE_GDC_URL}/files/${fileId}`; -const buildSequenceUrl = caseId => `${BASE_GDC_URL}/cases/${caseId}#files`; +const buildModelUrl = (caseId) => `${BASE_GDC_URL}/cases/${caseId}`; +const buildMafUrl = (fileId) => `${BASE_GDC_URL}/files/${fileId}`; +const buildSequenceUrl = (caseId) => `${BASE_GDC_URL}/cases/${caseId}#files`; export const addGenomicVariantsFromMaf = async (name, mafData, { filename, fileId }, caseId) => { const model = await Model.findOne({ name }); @@ -103,7 +103,7 @@ export const addGenomicVariantsFromMaf = async (name, mafData, { filename, fileI // Properties originally from Reference that are temporarily taken from MAF - Jon Eubank 2020-09-29 const gene = row.Hugo_Symbol; - const gene_biotype = titleCase(row.BIOTYPE.replace(/_/g, ' '), i => !i.includes('RNA')); + const gene_biotype = titleCase(row.BIOTYPE.replace(/_/g, ' '), (i) => !i.includes('RNA')); const name = `${row.Hugo_Symbol} ${aa_change}`; const synonyms = []; @@ -181,7 +181,7 @@ export const addGenomicVariantsFromMaf = async (name, mafData, { filename, fileI } await model.save(); - logger.audit( + logger.info( { model: model.name, genomic_variants: model.genomic_variants ? model.genomic_variants.length : 0, diff --git a/cms/src/helpers/matchedModels.js b/cms/src/helpers/matchedModels.js index a692807c..274644d9 100644 --- a/cms/src/helpers/matchedModels.js +++ b/cms/src/helpers/matchedModels.js @@ -1,18 +1,20 @@ +import _ from 'lodash'; + +import getLogger from '../logger.js'; import MatchedModels from '../schemas/matchedModels.js'; import Model from '../schemas/model.js'; + import { modelStatus } from './modelStatus.js'; -import _ from 'lodash'; -const { uniq } = _; -import getLogger from '../logger.js'; +const { uniq } = _; const logger = getLogger('helpers/matchedModels'); /* Worker methods, take models as inputs */ -const createMatchedModels = async models => { - const modelIds = models.map(model => model._id); +const createMatchedModels = async (models) => { + const modelIds = models.map((model) => model._id); const matchedModels = await MatchedModels.create({ models: modelIds }); - logger.audit( - { matchedModels, models: models.map(model => model.name) }, + logger.info( + { matchedModels, models: models.map((model) => model.name) }, 'matched model set created', 'Created matched models set', ); @@ -24,12 +26,12 @@ const createMatchedModels = async models => { ? modelStatus.unpublished : modelStatus.unpublishedChanges; model.save(); - logger.audit({ model: model.name }, 'model saved', 'Model updated to add matched model set'); + logger.info({ model: model.name }, 'model saved', 'Model updated to add matched model set'); } return matchedModels; }; -const clearModelFromSets = async model => { +const clearModelFromSets = async (model) => { // Get MatchedModels for this model if (!model.matchedModels) { @@ -42,7 +44,7 @@ const clearModelFromSets = async model => { if (matchedModels) { // only do operations on the matched set if we find it. if we didnt, no issue, we are removing this reference anyways matchedModels.models = matchedModels.models.filter( - modelId => modelId.toString() !== model._id.toString(), + (modelId) => modelId.toString() !== model._id.toString(), ); // We need to make sure that when these changes to our model are published, @@ -64,21 +66,21 @@ const clearModelFromSets = async model => { ? modelStatus.unpublished : modelStatus.unpublishedChanges; await otherModel.save(); - logger.audit( + logger.info( { model: otherModel.name }, 'model saved', 'Model updated to remove matched model set', ); } await MatchedModels.deleteOne({ _id: matchedModels._id }); - logger.audit( + logger.info( { matchedModels }, 'matched model set removed', 'Matched models set removed due to having 1 or fewer members after update', ); } else { await matchedModels.save(); - logger.audit({ matchedModels }, 'matched model set saved', 'Matched models set updated'); + logger.info({ matchedModels }, 'matched model set saved', 'Matched models set updated'); } } else { logger.warn( @@ -93,7 +95,7 @@ const clearModelFromSets = async model => { ? modelStatus.unpublished : modelStatus.unpublishedChanges; await model.save(); - logger.audit({ model: model.name }, 'model saved', 'Model updated to remove matched model set'); + logger.info({ model: model.name }, 'model saved', 'Model updated to remove matched model set'); return; }; @@ -103,7 +105,7 @@ const clearModelFromSets = async model => { * Clears the matchedModels for the given model and updates models in the old set to no longer include this model * @param {*} nameToAdd */ -const removeFromSet = async nameToRemove => { +const removeFromSet = async (nameToRemove) => { const model = await Model.findOne({ name: nameToRemove }); if (!model) { throw new Error(`Could not find model with the name: ${nameToRemove}.`); @@ -131,7 +133,8 @@ const connectWithMatchedModels = async (nameToAdd, setMemberName) => { ? await MatchedModels.findOne({ _id: modelToAdd.matchedModels }).populate('models') : null; const inCurrentSet = - currentMatchedModels && currentMatchedModels.models.find(model => model.name === setMemberName); + currentMatchedModels && + currentMatchedModels.models.find((model) => model.name === setMemberName); // Only continue if we have to, if: // a - the target model name (setMemberName) is not in the current matchedModels set @@ -152,7 +155,7 @@ const connectWithMatchedModels = async (nameToAdd, setMemberName) => { // Add model to the set matchedModels.models.push(modelToAdd._id); await matchedModels.save(); - logger.audit( + logger.info( { matchedModels, model: nameToAdd }, 'matched model set updated', 'Matched models set updated to add new model', @@ -165,7 +168,7 @@ const connectWithMatchedModels = async (nameToAdd, setMemberName) => { ? modelStatus.unpublished : modelStatus.unpublishedChanges; await modelToAdd.save(); - logger.audit( + logger.info( { model: modelToAdd.name }, 'model saved', 'Model updated to add to matched model set', @@ -181,7 +184,7 @@ const connectWithMatchedModels = async (nameToAdd, setMemberName) => { } } else { return { - models: currentMatchedModels.models.map(model => model._id), + models: currentMatchedModels.models.map((model) => model._id), _id: currentMatchedModels._id, __v: currentMatchedModels.__v, }; diff --git a/cms/src/helpers/uploadTemplate.js b/cms/src/helpers/uploadTemplate.js index f2a8d611..e82183ce 100644 --- a/cms/src/helpers/uploadTemplate.js +++ b/cms/src/helpers/uploadTemplate.js @@ -1,18 +1,21 @@ import { google } from 'googleapis'; import { getDictionary } from '../helpers/dictionary.js'; - -import { variantTypes, variantAssessmentType, variantExpressionLevel } from '../schemas/constants.js'; -import Variant from '../schemas/variant'; - +import { + variantTypes, + variantAssessmentType, + variantExpressionLevel, +} from '../schemas/constants.js'; +import Variant from '../schemas/variant.js'; import getLogger from '../logger.js'; + const logger = getLogger('helpers/uploadTemplate'); -const headerRowData = headerNames => ({ - values: headerNames.map(header => ({ userEnteredValue: { stringValue: header } })), +const headerRowData = (headerNames) => ({ + values: headerNames.map((header) => ({ userEnteredValue: { stringValue: header } })), }); -export const createModelUploadTemplate = async authClient => { +export const createModelUploadTemplate = async (authClient) => { const modelColumnHeaders = [ 'Name', 'Type', @@ -96,10 +99,7 @@ export const createModelUploadTemplate = async authClient => { const sheets = google.sheets({ version: 'v4', auth: authClient }); // Date formatted like: May-29-2020-13:51:42 - const dateStamp = Date() - .split(' ') - .splice(1, 4) - .join('-'); + const dateStamp = Date().split(' ').splice(1, 4).join('-'); const spreadsheet = { resource: { @@ -127,7 +127,7 @@ export const createModelUploadTemplate = async authClient => { }); const { spreadsheetId, spreadsheetUrl } = response.data; - const modelSheet = response.data.sheets.find(sheet => sheet.properties.title === 'Models'); + const modelSheet = response.data.sheets.find((sheet) => sheet.properties.title === 'Models'); // Add request for list validation for fields that have enum from data dictionary: const buildValidationRequest = (name, columnIndex) => ({ @@ -142,8 +142,8 @@ export const createModelUploadTemplate = async authClient => { condition: { type: 'ONE_OF_LIST', values: allDictionaryFields - .find(field => field.name === name) - .values.map(value => ({ userEnteredValue: value.value })), + .find((field) => field.name === name) + .values.map((value) => ({ userEnteredValue: value.value })), }, strict: 'true', showCustomUi: 'true', @@ -155,7 +155,7 @@ export const createModelUploadTemplate = async authClient => { for (const i in modelEnumNames) { const name = modelEnumNames[i]; - if (allDictionaryFields.find(field => field.name === name)) { + if (allDictionaryFields.find((field) => field.name === name)) { const request = buildValidationRequest(name, i); requests.push(request); } @@ -181,7 +181,7 @@ export const createModelUploadTemplate = async authClient => { }); }; -export const createVariantUploadTemplate = async authClient => { +export const createVariantUploadTemplate = async (authClient) => { const variantColumnHeaders = [ 'Model Name', 'Variant Name', @@ -191,15 +191,14 @@ export const createVariantUploadTemplate = async authClient => { ]; const variantData = await Variant.find({}); - const variantNames = variantData.map(variant => variant.name); + const variantNames = (variantData ?? []) + .map((variant) => variant.name) + .filter((name) => typeof name === 'string' && name.trim() !== ''); const sheets = google.sheets({ version: 'v4', auth: authClient }); // Date formatted like: May-29-2020-13:51:42 - const dateStamp = Date() - .split(' ') - .splice(1, 4) - .join('-'); + const dateStamp = Date().split(' ').splice(1, 4).join('-'); const spreadsheet = { resource: { @@ -227,7 +226,7 @@ export const createVariantUploadTemplate = async authClient => { }); const { spreadsheetId, spreadsheetUrl } = response.data; - const variantSheet = response.data.sheets.find(sheet => sheet.properties.title === 'Variants'); + const variantSheet = response.data.sheets.find((sheet) => sheet.properties.title === 'Variants'); const buildValidationRequest = (columnIndex, values) => ({ setDataValidation: { @@ -240,7 +239,9 @@ export const createVariantUploadTemplate = async authClient => { rule: { condition: { type: 'ONE_OF_LIST', - values: values.map(value => ({ userEnteredValue: value })), + values: values + .filter((value) => value != null && String(value).trim() !== '') + .map((value) => ({ userEnteredValue: value })), }, strict: 'true', showCustomUi: 'true', diff --git a/cms/src/helpers/validation.js b/cms/src/helpers/validation.js index a70e7408..038db954 100644 --- a/cms/src/helpers/validation.js +++ b/cms/src/helpers/validation.js @@ -3,12 +3,12 @@ import { ValidationError } from 'yup'; export const runYupValidatorFailSlow = (validator, data) => { - const validatePromises = data.map(p => - validator.validate(p, { abortEarly: false }).catch(Error => Error), + const validatePromises = data.map((p) => + validator.validate(p, { abortEarly: false }).catch((Error) => Error), ); - return Promise.all(validatePromises).then(results => - results.map(result => { + return Promise.all(validatePromises).then((results) => + results.map((result) => { if (!(result instanceof ValidationError)) { return { success: true, @@ -37,12 +37,12 @@ export const runYupValidatorFailSlow = (validator, data) => { }; export const runYupValidatorFailFast = (validator, data) => { - const validatePromises = data.map(p => - validator.validate(p, { abortEarly: false, strict: false }).catch(Error => Error), + const validatePromises = data.map((p) => + validator.validate(p, { abortEarly: false, strict: false }).catch((Error) => Error), ); - return Promise.all(validatePromises).then(results => { - const failed = results.filter(result => result instanceof Error); + return Promise.all(validatePromises).then((results) => { + const failed = results.filter((result) => result instanceof Error); if (failed.length > 0) { const errors = { validationErrors: failed.map(({ value, inner }) => ({ @@ -62,5 +62,5 @@ export const runYupValidatorFailFast = (validator, data) => { }); }; -const getErrorDetails = validationResult => - validationResult.inner.flatMap(({ errors, path }) => errors.map(error => `${path}: ${error}`)); +const getErrorDetails = (validationResult) => + validationResult.inner.flatMap(({ errors, path }) => errors.map((error) => `${path}: ${error}`)); diff --git a/cms/src/hooks.js b/cms/src/hooks.js index 15c00c5d..4389bec4 100644 --- a/cms/src/hooks.js +++ b/cms/src/hooks.js @@ -1,14 +1,15 @@ -import { publishModel } from './services/elastic-search/publish.js'; -import { unpublishModel } from './services/elastic-search/unpublish.js'; +import _ from 'lodash'; + +import { publishModel } from './services/search-client/publish.js'; +import { unpublishModel } from './services/search-client/unpublish.js'; import { modelStatus, runYupValidatorFailFast } from './helpers/index.js'; import { deleteImage } from './routes/images.js'; import { getSaveValidation } from './validation/model.js'; import { getLoggedInUser } from './helpers/authorizeUserAccess.js'; import userValidation from './validation/user.js'; -import _ from 'lodash'; -const { transform } = _; - import getLogger from './logger.js'; + +const { transform } = _; const logger = getLogger('hooks'); export const validateYup = (req, res, next) => { @@ -92,7 +93,7 @@ export const outputFn = async (req, res, next) => { }; export const postCreate = async (req, res, next) => { - logger.audit({ model: req.erm.result }, 'model created', 'Model created in mongo'); + logger.info({ model: req.erm.result }, 'model created', 'Model created in mongo'); return next(); }; @@ -106,7 +107,7 @@ export const postUpdate = async (req, res, next) => { }, } = req; - logger.audit({ model: modelName }, 'model saved', 'Model saved in mongo'); + logger.info({ model: modelName }, 'model saved', 'Model saved in mongo'); // Model updates that contain the status key we // treat as being a change in status and trigger diff --git a/cms/src/index.js b/cms/src/index.js index aa150b1c..add582b7 100644 --- a/cms/src/index.js +++ b/cms/src/index.js @@ -1,15 +1,27 @@ -// @ts-nocheck import 'babel-polyfill'; +import bodyParser from 'body-parser'; +import cors from 'cors'; import express from 'express'; +import { serve as restify } from 'express-restify-mongoose'; +import helmet from 'helmet'; import { Server } from 'http'; -import cors from 'cors'; -import mongoose from 'mongoose'; -import bodyParser from 'body-parser'; import methodOverride from 'method-override'; -import restify from 'express-restify-mongoose'; +import mongoose from 'mongoose'; import pino from 'pino-http'; -import helmet from 'helmet'; +import pm2Config from './../pm2.config.js'; + +import isUserAuthorized, { USER_EMAIL, getLoggedInUser } from './helpers/authorizeUserAccess.js'; +import { + preUpdate, + validateYup, + preModelDelete, + postUpdate, + postCreate, + outputFn, + validateUserRequest, +} from './hooks.js'; +import getLogger from './logger.js'; import { data_sync_router } from './routes/sync-data.js'; import { actionRouter, @@ -23,23 +35,21 @@ import { publishRouter, authRouter, } from './routes/index.js'; -import { - preUpdate, - validateYup, - preModelDelete, - postUpdate, - postCreate, - outputFn, - validateUserRequest, -} from './hooks.js'; import Model from './schemas/model.js'; import MatchedModels from './schemas/matchedModels.js'; import User from './schemas/user.js'; -import isUserAuthorized, { USER_EMAIL, getLoggedInUser } from './helpers/authorizeUserAccess.js'; -import getLogger from './logger.js'; -const logger = getLogger('root'); +const pm2Env = process.env.ENV; +if (!pm2Env) { + throw new Error('No ENV value provided!'); +} +const pm2ConfigGeneric = + (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0].env) || {}; +const pm2ConfigForEnv = + (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0][`env_${pm2Env}`]) || {}; +export const pm2 = { ...pm2ConfigGeneric, ...pm2ConfigForEnv }; +const logger = getLogger('root'); const port = process.env.PORT || 8080; const app = express(); const modelRouter = express.Router(); @@ -86,7 +96,7 @@ if (process.env.AUTH_ENABLED !== 'false') { app.use(pino({ customProps: (req) => ({ user: getLoggedInUser(req).user_email }) })); // configure endpoints -restify.serve(modelRouter, Model, { +restify(modelRouter, Model, { preCreate: validateYup, postCreate, preUpdate, @@ -96,10 +106,10 @@ restify.serve(modelRouter, Model, { idProperty: 'name', }); -restify.serve(matchedModelsRestifyRouter, MatchedModels); +restify(matchedModelsRestifyRouter, MatchedModels); // configure endpoints -restify.serve(userRouter, User, { +restify(userRouter, User, { preCreate: validateUserRequest, preUpdate: validateUserRequest, }); diff --git a/cms/src/pm2.js b/cms/src/pm2.js new file mode 100644 index 00000000..3299fd36 --- /dev/null +++ b/cms/src/pm2.js @@ -0,0 +1,14 @@ +import pm2Config from './../pm2.config.js'; + +const pm2Env = process.env.ENV; +if (!pm2Env) { + throw new Error('No ENV value provided!'); +} +const pm2ConfigGeneric = + (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0].env) || {}; +const pm2ConfigForEnv = + (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0][`env_${pm2Env}`]) || {}; + +const pm2 = { ...pm2ConfigGeneric, ...pm2ConfigForEnv }; + +export default pm2; diff --git a/cms/src/routes/action.js b/cms/src/routes/action.js index 04a7d6b3..3d972142 100644 --- a/cms/src/routes/action.js +++ b/cms/src/routes/action.js @@ -4,8 +4,8 @@ import express from 'express'; import Model from '../schemas/model.js'; import getPublishValidation from '../validation/model.js'; import { runYupValidatorFailFast } from '../helpers/index.js'; -import { publishModel } from '../services/elastic-search/publish.js'; -import { unpublishModel } from '../services/elastic-search/unpublish.js'; +import { publishModel } from '../services/search-client/publish.js'; +import { unpublishModel } from '../services/search-client/unpublish.js'; import { backupFields } from '../schemas/descriptions/modelVariant.js'; import csvStream from '../helpers/streamAsCSV.js'; @@ -22,10 +22,10 @@ actionRouter.post('/publish/:name', async (req, res) => { name, }) .populate('variants.variant') - .then(models => runYupValidatorFailFast(validation, models)) - .then(() => publishModel({ name })) + .then((models) => runYupValidatorFailFast(validation, models)) + .then(async () => await publishModel({ name })) .then(() => res.json({ success: `${name} has been successfully published` })) - .catch(error => { + .catch((error) => { logger.error(error); res.status(500).json({ error: error, diff --git a/cms/src/routes/bulk.js b/cms/src/routes/bulk.js index e7e5c000..912ad830 100644 --- a/cms/src/routes/bulk.js +++ b/cms/src/routes/bulk.js @@ -1,30 +1,30 @@ // @ts-check import express from 'express'; + import Model from '../schemas/model.js'; import getPublishValidation from '../validation/model.js'; import { runYupValidatorFailSlow, modelStatus } from '../helpers/index.js'; -import { indexOneToES, indexMatchedModelsToES } from '../services/elastic-search/publish.js'; -import { unpublishManyFromES } from '../services/elastic-search/unpublish.js'; +import { indexOneToES, indexMatchedModelsToES } from '../services/search-client/publish.js'; +import { unpublishManyFromES } from '../services/search-client/unpublish.js'; import csvStream from '../helpers/streamAsCSV.js'; import { backupFields } from '../schemas/descriptions/model.js'; -import { updateGeneSearchIndicies } from '../services/elastic-search/genomicVariants.js'; +import { updateGeneSearchIndicies } from '../services/search-client/genomicVariants.js'; +import getLogger from '../logger.js'; -import getLogger from '../logger'; const logger = getLogger('routes/bulk'); - const bulkRouter = express.Router(); bulkRouter.post('/publish', async (req, res) => { const validation = await getPublishValidation(); - let validationErrors; + let validationErrors = []; // Validate models for publishing Model.find({ _id: { $in: req.body }, }) .populate('variants.variant') - .then(models => runYupValidatorFailSlow(validation, models)) - .then(validated => { + .then((models) => runYupValidatorFailSlow(validation, models)) + .then((validated) => { const validModelNames = validated .filter(({ success }) => success) .map(({ result: { name } }) => name); @@ -34,7 +34,7 @@ bulkRouter.post('/publish', async (req, res) => { return validModelNames; }) - .then(async validModelNames => { + .then(async (validModelNames) => { for (const name of validModelNames) { try { await indexOneToES({ name }); @@ -60,14 +60,14 @@ bulkRouter.post('/publish', async (req, res) => { { $set: { variants_modified: false } }, ); } - res.json({ + return res.json({ success: `${req.body.length - validationErrors.length} models published`, errors: validationErrors, }); }) - .catch(error => { + .catch((error) => { logger.error(error); - res.status(500).json({ + return res.status(500).json({ error: error, }); }); @@ -76,60 +76,57 @@ bulkRouter.post('/publish', async (req, res) => { bulkRouter.post('/unpublish', async (req, res) => { let deleteCount = 0; - Model.find({ _id: { $in: req.body } }) - .then(models => unpublishManyFromES(models.map(({ name }) => name))) - .then(result => { - deleteCount = result.deleted; - return Model.updateMany( - { - _id: { $in: req.body }, - }, - { status: modelStatus.unpublished }, - ); - }) - .then(async () => { - for (const _id of req.body) { - // Now that everything has been published, lets make sure all the matched models for these are also updated in ES - await indexMatchedModelsToES({ _id }); - } - await updateGeneSearchIndicies(); - }) - .then(() => res.json({ success: `${deleteCount} models unpublished` })) - .catch(error => { - logger.error(error); - res.status(500).json({ - error: error, - }); + try { + const models = await Model.find({ _id: { $in: req.body } }); + const result = await unpublishManyFromES(models.map(({ name }) => name)); + deleteCount = result.body.deleted; + await Model.updateMany( + { + _id: { $in: req.body }, + }, + { status: modelStatus.unpublished }, + ); + + for (const _id of req.body) { + // Now that everything has been published, lets make sure all the matched models for these are also updated in ES + await indexMatchedModelsToES({ _id }); + } + await updateGeneSearchIndicies(); + return res.json({ success: `${deleteCount} models unpublished` }); + } catch (error) { + logger.error(error); + return res.status(500).json({ + error: error, }); + } }); bulkRouter.post('/delete', async (req, res) => { - Model.find({ _id: { $in: req.body } }) - .then(models => { - const modelsToUnpublish = models - .filter(({ status }) => status !== modelStatus.unpublished) - .map(({ name }) => name); - return unpublishManyFromES(modelsToUnpublish); - }) - .then(async () => { - for (const _id of req.body) { - // Now that everything has been published, lets make sure all the matched models for these are also updated in ES - await indexMatchedModelsToES({ _id }); - } - await updateGeneSearchIndicies(); - }) - .then(() => - Model.deleteMany({ - _id: { $in: req.body }, - }), - ) - .then(() => res.json({ success: `${req.body.length} models deleted` })) - .catch(error => { - logger.error(error); - res.status(500).json({ - error: error, - }); + try { + const models = await Model.find({ _id: { $in: req.body } }); + const modelsToUnpublish = models + .filter(({ status }) => status !== modelStatus.unpublished) + .map(({ name }) => name); + await unpublishManyFromES(modelsToUnpublish); + + for (const _id of req.body) { + // Now that everything has been published, lets make sure all the matched models for these are also updated in ES + await indexMatchedModelsToES({ _id }); + } + + await updateGeneSearchIndicies(); + + await Model.deleteMany({ + _id: { $in: req.body }, + }); + + return res.json({ success: `${req.body.length} models deleted` }); + } catch (error) { + logger.error(error); + return res.status(500).json({ + error: error, }); + } }); bulkRouter.get('/backup', async (req, res) => { diff --git a/cms/src/routes/dictionary.js b/cms/src/routes/dictionary.js index 0794935e..7862779d 100644 --- a/cms/src/routes/dictionary.js +++ b/cms/src/routes/dictionary.js @@ -70,7 +70,7 @@ draftRouter.patch('/', async (req, res) => { } const draftDoc = await DictionaryHelper.getDictionaryDraft(); - const draft = draftDoc.fields.find(i => i.name === field); + const draft = draftDoc.fields.find((i) => i.name === field); if (!draft) { res.status(400).json({ err: `No dictionary field found named: ${field}` }); @@ -95,7 +95,7 @@ draftRouter.patch('/', async (req, res) => { return; } - const parentValue = draft.values.find(val => + const parentValue = draft.values.find((val) => val.original ? val.original === parent : val.value === parent, ); @@ -104,14 +104,14 @@ draftRouter.patch('/', async (req, res) => { return; } - const dependent = parentValue.dependents.find(dep => dep.name === dependentName); + const dependent = parentValue.dependents.find((dep) => dep.name === dependentName); if (!dependent) { res.status(400).json({ err: `Parent value has no values for this dependent name` }); return; } - const editValue = dependent.values.find(val => + const editValue = dependent.values.find((val) => val.original ? val.original === original : val.value === original, ); @@ -126,7 +126,7 @@ draftRouter.patch('/', async (req, res) => { DictionaryHelper.editValue(editValue, original, updated); } else { // handle basic case - const editValue = draft.values.find(val => + const editValue = draft.values.find((val) => val.original ? val.original === original : val.value === original, ); if (!editValue) { @@ -142,7 +142,7 @@ draftRouter.patch('/', async (req, res) => { draft.stats = DictionaryHelper.countDraftStats(draft); await draftDoc.save(); - logger.audit( + logger.info( { field, parent, dependentName, original, updated }, 'draft updated', 'Dictionary draft value edited', @@ -177,7 +177,7 @@ draftRouter.post('/', async (req, res) => { } const draftDoc = await DictionaryHelper.getDictionaryDraft(); - const draft = draftDoc.fields.find(i => i.name === field); + const draft = draftDoc.fields.find((i) => i.name === field); if (!draft) { res.status(400).json({ err: `No dictionary field found named: ${field}` }); @@ -202,7 +202,7 @@ draftRouter.post('/', async (req, res) => { return; } - const parentValue = draft.values.find(val => + const parentValue = draft.values.find((val) => val.original ? val.original === parent : val.value === parent, ); @@ -211,13 +211,13 @@ draftRouter.post('/', async (req, res) => { return; } - let dependent = parentValue.dependents.find(dep => dep.name === dependentName); + let dependent = parentValue.dependents.find((dep) => dep.name === dependentName); if (!dependent) { dependent = { name: dependentName, // displayName has a replace that does: To Title Case - displayName: dependentName.replace(/\w\S*/g, function(txt) { + displayName: dependentName.replace(/\w\S*/g, function (txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); }), values: [], @@ -244,7 +244,7 @@ draftRouter.post('/', async (req, res) => { } draft.stats = DictionaryHelper.countDraftStats(draft); await draftDoc.save(); - logger.audit( + logger.info( { field, parent, dependentName, value }, 'draft updated', 'Dictionary draft value added', @@ -277,7 +277,7 @@ draftRouter.post('/remove', async (req, res) => { } const draftDoc = await DictionaryHelper.getDictionaryDraft(); - const draft = draftDoc.fields.find(i => i.name === field); + const draft = draftDoc.fields.find((i) => i.name === field); if (!draft) { res.status(400).json({ err: `No dictionary field found named: ${field}` }); @@ -302,7 +302,7 @@ draftRouter.post('/remove', async (req, res) => { return; } - const parentValue = draft.values.find(val => + const parentValue = draft.values.find((val) => val.original ? val.original === parent : val.value === parent, ); @@ -311,13 +311,13 @@ draftRouter.post('/remove', async (req, res) => { return; } - let dependent = parentValue.dependents.find(dep => dep.name === dependentName); + let dependent = parentValue.dependents.find((dep) => dep.name === dependentName); if (!dependent) { dependent = { name: dependentName, // displayName has a replace that does: To Title Case - displayName: dependentName.replace(/\w\S*/g, function(txt) { + displayName: dependentName.replace(/\w\S*/g, function (txt) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); }), values: [], @@ -345,7 +345,7 @@ draftRouter.post('/remove', async (req, res) => { } draft.stats = DictionaryHelper.countDraftStats(draft); await draftDoc.save(); - logger.audit( + logger.info( { field, parent, dependentName, value }, 'draft updated', 'Dictionary draft new value removed', diff --git a/cms/src/routes/health.js b/cms/src/routes/health.js index c11b7793..c7cdb9bc 100644 --- a/cms/src/routes/health.js +++ b/cms/src/routes/health.js @@ -1,7 +1,7 @@ import express from 'express'; import _ from 'lodash'; -import client from '../services/elastic-search/common/client.js'; +import client from '../services/search-client/client.js'; import Model from '../schemas/model.js'; import { testS3Connection } from '../services/s3/index.js'; @@ -42,12 +42,12 @@ healthRouter.get('/db', async (req, res) => { healthRouter.get('/s3', async (req, res) => { try { await testS3Connection() - .then(data => + .then((data) => res .status(200) .json({ status: 200, response: `Connected to S3 successfully: ${JSON.stringify(data)}` }), ) - .catch(err => + .catch((err) => res.status(err.statusCode).json({ error: 'Error connecting to S3', response: err.code }), ); } catch (e) { diff --git a/cms/src/routes/publish.js b/cms/src/routes/publish.js index 950e00cb..0d6fc70b 100644 --- a/cms/src/routes/publish.js +++ b/cms/src/routes/publish.js @@ -64,11 +64,10 @@ publishRouter.get('/status', async (req, res) => { logger.debug(`Fetching Publisher status...`); const status = Publisher.getStatus(); - - res.status(200).json({ ...status }); + return res.status(200).json({ ...status }); } catch (error) { logger.error(error, `Error occurred during Publisher status fetch`); - res.status(500).json({ + return res.status(500).json({ success: false, error: error, }); @@ -154,11 +153,10 @@ publishRouter.post('/acknowledge/bulk', async (req, res) => { logger.debug(`Starting bulk acknowledge for models: ${models}`); const acknowledged = Publisher.acknowledgeBulk(models); - - res.status(200).json({ acknowledged, success: true }); + return res.status(200).json({ acknowledged, success: true }); } catch (error) { logger.error(error, `Error occurred during bulk model acknowledge`); - res.status(500).json({ + return res.status(500).json({ success: false, error: error, }); @@ -172,10 +170,10 @@ publishRouter.post('/acknowledge/:name', async (req, res) => { const acknowledged = Publisher.acknowledge(name); - res.status(200).json({ acknowledged, success: true }); + return res.status(200).json({ acknowledged, success: true }); } catch (error) { logger.error(error, `Error occurred during publish status acknowledgement for model ${name}`); - res.status(500).json({ + return res.status(500).json({ success: false, error: error, }); @@ -188,10 +186,10 @@ publishRouter.post('/stop/all', async (req, res) => { const stopped = await Publisher.stopBulkPublish(); - res.status(200).json({ stopped, success: true }); + return res.status(200).json({ stopped, success: true }); } catch (error) { logger.error(error, `Error occurred during bulk publish stop`); - res.status(500).json({ + return res.status(500).json({ success: false, error: error, }); @@ -205,10 +203,10 @@ publishRouter.post('/stop/:name', async (req, res) => { const stopped = await Publisher.stopPublish(name); - res.status(200).json({ stopped, success: true }); + return res.status(200).json({ stopped, success: true }); } catch (error) { logger.error(error, `Error occurred during publish stop for model ${name}`); - res.status(500).json({ + return res.status(500).json({ success: false, error: error, }); diff --git a/cms/src/routes/sync-data.js b/cms/src/routes/sync-data.js index dda5d308..bd72d62e 100644 --- a/cms/src/routes/sync-data.js +++ b/cms/src/routes/sync-data.js @@ -2,18 +2,23 @@ import express from 'express'; import _ from 'lodash'; -const { unionWith, uniqWith, isEqual } = _; +import { getLoggedInUser } from '../helpers/authorizeUserAccess.js'; +import { + modelStatus, + ensureAuth, + computeModelStatus, + runYupValidatorFailSlow, +} from '../helpers/index.js'; +import getLogger from '../logger.js'; import { toExcelHeaders, toExcelRowNumber } from '../schemas/constants.js'; import Model, { ModelSchema } from '../schemas/model.js'; import Variant from '../schemas/variant.js'; +import { getSheetData, typeToParser, NAtoNull } from '../services/import/SheetsToMongo.js'; import { getSaveValidation } from '../validation/model.js'; import { modelVariantUploadSchema } from '../validation/variant.js'; -import { modelStatus, ensureAuth, computeModelStatus, runYupValidatorFailSlow } from '../helpers/index.js'; -import { getSheetData, typeToParser, NAtoNull } from '../services/import/SheetsToMongo.js'; -import { getLoggedInUser } from '../helpers/authorizeUserAccess.js'; -import getLogger from '../logger.js'; +const { unionWith, uniqWith, isEqual } = _; const logger = getLogger('routes/sync-data'); export const data_sync_router = express.Router(); @@ -23,9 +28,9 @@ data_sync_router.get('/sheets-data/:spreadsheetId/:sheetId', async (req, res) => const { spreadsheetId, sheetId } = req.params; ensureAuth(req) - .then(authClient => getSheetData({ authClient, spreadsheetId, sheetId })) - .then(data => res.json(data)) - .catch(error => { + .then((authClient) => getSheetData({ authClient, spreadsheetId, sheetId })) + .then((data) => res.json(data)) + .catch((error) => { logger.error(error, 'Unexpected error occurred while reading Google Sheets'); res.status(500).json({ message: `An unexpected error occurred while trying to read Google Sheet ID: ${sheetId}, ${error}`, @@ -37,8 +42,8 @@ data_sync_router.get('/wrangle-cde/:spreadsheetId/:sheetId', async (req, res) => const { spreadsheetId, sheetId } = req.params; ensureAuth(req) - .then(authClient => getSheetData({ authClient, spreadsheetId, sheetId })) - .then(data => { + .then((authClient) => getSheetData({ authClient, spreadsheetId, sheetId })) + .then((data) => { const transformed = Object.entries(toExcelRowNumber).reduce((acc, [type, rowNumber]) => { return { ...acc, @@ -54,7 +59,7 @@ data_sync_router.get('/wrangle-cde/:spreadsheetId/:sheetId', async (req, res) => }, {}); return res.json(transformed); }) - .catch(error => { + .catch((error) => { logger.error( { error, sheetId, spreadsheetId }, 'Unexpected error while reading Google Sheet', @@ -65,7 +70,8 @@ data_sync_router.get('/wrangle-cde/:spreadsheetId/:sheetId', async (req, res) => }); }); -const normalizeOption = option => (option === 'true' ? true : option === 'false' ? false : option); +const normalizeOption = (option) => + option === 'true' ? true : option === 'false' ? false : option; data_sync_router.get('/bulk-models/:spreadsheetId/:sheetId', async (req, res) => { const { spreadsheetId, sheetId } = req.params; @@ -75,13 +81,13 @@ data_sync_router.get('/bulk-models/:spreadsheetId/:sheetId', async (req, res) => overwrite = normalizeOption(overwrite); ensureAuth(req) - .then(authClient => getSheetData({ authClient, spreadsheetId, sheetId })) - .then(async data => { + .then((authClient) => getSheetData({ authClient, spreadsheetId, sheetId })) + .then(async (data) => { const parsed = data .filter(({ name }) => name) - .map(d => + .map((d) => Object.keys(d) - .filter(key => ModelSchema.paths[key]) + .filter((key) => ModelSchema.paths[key]) .reduce( (acc, key) => ({ ...acc, @@ -94,7 +100,7 @@ data_sync_router.get('/bulk-models/:spreadsheetId/:sheetId', async (req, res) => const validation = await getSaveValidation(); return runYupValidatorFailSlow(validation, parsed); }) - .then(validated => { + .then((validated) => { const savePromises = validated .filter(({ success }) => success) .map(async ({ result }) => { @@ -134,35 +140,31 @@ data_sync_router.get('/bulk-models/:spreadsheetId/:sheetId', async (req, res) => }, ) .then(() => resolve({ status: 'updated', doc: result.name })) - .catch(error => { + .catch((error) => { logger.error( { error, model: result.name }, 'Unexpected error occurred while updating one model in bulk update', ); reject({ - message: `An unexpected error occurred while updating model: ${ - result.name - }, Error: ${error}`, + message: `An unexpected error occurred while updating model: ${result.name}, Error: ${error}`, }); }); }); } - return new Promise(resolve => resolve({ status: 'unchanged', doc: result.name })); //no fields modified, do nothing + return new Promise((resolve) => resolve({ status: 'unchanged', doc: result.name })); //no fields modified, do nothing } else { return new Promise((resolve, reject) => { const newModel = new Model(addUserEmail(req, result)); newModel .save() .then(() => resolve({ status: 'new', doc: result.name })) - .catch(error => { + .catch((error) => { logger.error( { error, model: result.name }, 'Unexpected error occurred while creating model during bulk data sync', ); reject({ - message: `An unexpected error occurred while creating model: ${ - result.name - }, Error: ${error}`, + message: `An unexpected error occurred while creating model: ${result.name}, Error: ${error}`, }); }); }); @@ -174,7 +176,7 @@ data_sync_router.get('/bulk-models/:spreadsheetId/:sheetId', async (req, res) => .map(({ errors }) => errors); return Promise.all(savePromises) - .then(saveResults => + .then((saveResults) => res.json({ result: saveResults.reduce( (finalResponse, saveResult) => { @@ -186,11 +188,11 @@ data_sync_router.get('/bulk-models/:spreadsheetId/:sheetId', async (req, res) => ), }), ) - .catch(error => { + .catch((error) => { throw error; }); }) - .catch(error => { + .catch((error) => { logger.error(error, 'Unexpected error occured in bulk upload'); res.status(500).json({ error: error.details || 'Unknown error occurred.' }); }); @@ -204,26 +206,26 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn overwrite = normalizeOption(overwrite); ensureAuth(req) - .then(authClient => getSheetData({ authClient, spreadsheetId, sheetId })) - .then(data => - data.map(modelVariantUpload => { + .then((authClient) => getSheetData({ authClient, spreadsheetId, sheetId })) + .then((data) => + data.map((modelVariantUpload) => { // Remove all null / undefined / empty Object.keys(modelVariantUpload).forEach( - key => !modelVariantUpload[key] && delete modelVariantUpload[key], + (key) => !modelVariantUpload[key] && delete modelVariantUpload[key], ); return modelVariantUpload; }), ) - .then(data => runYupValidatorFailSlow(modelVariantUploadSchema, data)) - .then(validated => + .then((data) => runYupValidatorFailSlow(modelVariantUploadSchema, data)) + .then((validated) => Promise.all( - validated.map(validatedVariant => { + validated.map((validatedVariant) => { if (validatedVariant.success) { const variantData = validatedVariant.result; return Variant.findOne({ name: variantData.variant_name, type: variantData.variant_type, - }).then(variantResult => { + }).then((variantResult) => { // If no variant found return an error in // the same format as validation errors if (!variantResult) { @@ -232,9 +234,7 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn errors: { name: variantData.variant_name || 'Unknown', details: [ - `No variant found matching "${variantData.variant_name}" and "${ - variantData.variant_type - }" in database.`, + `No variant found matching "${variantData.variant_name}" and "${variantData.variant_type}" in database.`, ], }, }; @@ -254,12 +254,12 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn }); } else { // Return the unsuccessfull validation error like normal - return new Promise(resolve => resolve(validatedVariant)); + return new Promise((resolve) => resolve(validatedVariant)); } }), ), ) - .then(populatedVariants => { + .then((populatedVariants) => { // Sort the modelVariant relations by model_name const mappedModelVariants = populatedVariants .filter(({ success }) => success) @@ -284,7 +284,7 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn }, {}); // Process all successfully populated variants as normal - const savePromises = Object.keys(mappedModelVariants).map(async model_name => { + const savePromises = Object.keys(mappedModelVariants).map(async (model_name) => { // Upload set for the model we are operating on const uploadedModelVariants = uniqWith(mappedModelVariants[model_name], isEqual); @@ -341,21 +341,19 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn variants: allowedUpdates, }), ) - .catch(error => { + .catch((error) => { logger.error( { error, model: model.name }, 'Unexpected error occured while updating model in bulk data sync', ); reject({ - message: `An unexpected error occurred while updating model: ${ - model.name - }, Error: ${error}`, + message: `An unexpected error occurred while updating model: ${model.name}, Error: ${error}`, variants: allowedUpdates, }); }); }); } else { - return new Promise(resolve => + return new Promise((resolve) => resolve({ status: 'unchanged', doc: model.name, variants: uploadedModelVariants }), ); } @@ -367,8 +365,8 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn .map(({ errors }) => errors); return Promise.all(savePromises) - .then(saveResults => - res.json({ + .then((saveResults) => { + return res.json({ result: saveResults.reduce( (finalResponse, saveResult) => { const { status, doc, variants } = saveResult; @@ -377,13 +375,13 @@ data_sync_router.get('/attach-variants/:spreadsheetId/:sheetId/:modelName', asyn }, { unchanged: [], updated: [], new: [], errors }, ), - }), - ) - .catch(error => { + }); + }) + .catch((error) => { throw error; }); }) - .catch(error => { + .catch((error) => { logger.error(error, 'Unexpected error occurred during Sync Data'); return res.status(500).json({ error: error instanceof Error ? error.message : error }); }); diff --git a/cms/src/schemas/genes.js b/cms/src/schemas/genes.js deleted file mode 100644 index f20d9fd2..00000000 --- a/cms/src/schemas/genes.js +++ /dev/null @@ -1,26 +0,0 @@ -import mongoose from 'mongoose'; - -import mongooseElasticsearch from 'mongoose-elasticsearch-xp'; -import elasticClient from '../services/elastic-search/common/client.js'; - -const GeneSchema = new mongoose.Schema( - { - _gene_id: { type: String, unique: true, required: true, es_indexed: true }, - symbol: { type: String, unique: true, required: true, es_indexed: true }, - ensemble_id: { type: String, unique: true, required: true, es_indexed: true }, - name: { type: String, es_indexed: true }, - synonyms: { type: [String], es_indexewd: true }, - biotype: { type: String, es_indexed: false }, - }, - { - collection: 'genesReference', - }, -); - -GeneSchema.plugin(mongooseElasticsearch.v7, { - client: elasticClient, - index: 'genes', - type: '_doc', -}); - -export default mongoose.model('Gene', GeneSchema); diff --git a/cms/src/schemas/model.js b/cms/src/schemas/model.js index 1995567e..282d6a0f 100644 --- a/cms/src/schemas/model.js +++ b/cms/src/schemas/model.js @@ -1,6 +1,6 @@ import mongoose from 'mongoose'; import _ from 'lodash'; -const { flatten, uniq } = _; + import { modelStatus } from '../helpers/modelStatus.js'; import getLogger from '../logger.js'; @@ -55,188 +55,64 @@ const GenomicVariant = new mongoose.Schema({ synonyms: { type: [String] }, }); -export const ModelSchema = new mongoose.Schema( - { - name: { type: String, unique: true, required: true, es_indexed: true }, - type: { type: String, es_indexed: true }, - growth_rate: { type: Number, set: deleteEmptyStrings, es_indexed: true }, - split_ratio: { type: String, es_indexed: true }, - time_to_split: { type: String, es_indexed: true }, - gender: { type: String, es_indexed: true }, - race: { type: String, es_indexed: true }, - age_at_diagnosis: { type: Number, es_indexed: true }, - age_at_sample_acquisition: { type: Number, es_indexed: true }, - date_of_availability: { type: Date, es_indexed: true }, - primary_site: { type: String, es_indexed: true }, - tnm_stage: { type: String, es_indexed: true }, - neoadjuvant_therapy: { type: String, es_indexed: true }, - chemotherapeutic_drugs: { type: Boolean, es_indexed: true }, - disease_status: { type: String, es_indexed: true }, - vital_status: { type: String, es_indexed: true }, - therapy: { type: [String], es_indexed: true }, - molecular_characterizations: { type: [String], es_indexed: true }, - tissue_type: { type: String, es_indexed: true }, - clinical_tumor_diagnosis: { type: String, es_indexed: false }, - histological_type: { type: String, set: deleteEmptyStrings, es_indexed: false }, - clinical_stage_grouping: { type: String, set: deleteEmptyStrings, es_indexed: false }, - site_of_sample_acquisition: { type: String, set: deleteEmptyStrings, es_indexed: false }, - tumor_histological_grade: { type: String, set: deleteEmptyStrings, es_indexed: false }, - licensing_required: { type: Boolean, es_indexed: true }, - distributor_part_number: { type: String, es_indexed: true }, - source_model_url: { type: String, es_indexed: true }, - source_sequence_url: { type: String, es_indexed: true }, - somatic_maf_url: { type: String, es_indexed: true }, - proteomics_url: { type: String, es_indexed: true }, - expanded: { type: Boolean, es_indexed: true }, - files: { type: [FilesSchema], es_indexed: true }, - variants: { type: [VariantExpression], es_indexed: true }, - variants_modified: { type: Boolean, es_indexed: false, default: false }, - genomic_variants: { type: [GenomicVariant], es_indexed: true }, - gene_metadata: { - type: GeneMetadata, - es_indexed: true, - }, - matchedModels: { - type: mongoose.Schema.Types.ObjectId, - ref: 'MatchedModels', - es_indexed: false, - }, - updateOldMatchesOnPublish: { - type: [String], - es_indexed: false, - }, - status: { - type: String, - enum: [modelStatus.unpublished, modelStatus.published, modelStatus.unpublishedChanges], - default: modelStatus.unpublished, - es_indexed: false, - }, - updatedBy: { type: String, es_indexed: false }, +export const ModelSchema = new mongoose.Schema({ + name: { type: String, unique: true, required: true, es_indexed: true }, + type: { type: String, es_indexed: true }, + growth_rate: { type: Number, set: deleteEmptyStrings, es_indexed: true }, + split_ratio: { type: String, es_indexed: true }, + time_to_split: { type: String, es_indexed: true }, + gender: { type: String, es_indexed: true }, + race: { type: String, es_indexed: true }, + age_at_diagnosis: { type: Number, es_indexed: true }, + age_at_sample_acquisition: { type: Number, es_indexed: true }, + date_of_availability: { type: Date, es_indexed: true }, + primary_site: { type: String, es_indexed: true }, + tnm_stage: { type: String, es_indexed: true }, + neoadjuvant_therapy: { type: String, es_indexed: true }, + chemotherapeutic_drugs: { type: Boolean, es_indexed: true }, + disease_status: { type: String, es_indexed: true }, + vital_status: { type: String, es_indexed: true }, + therapy: { type: [String], es_indexed: true }, + molecular_characterizations: { type: [String], es_indexed: true }, + tissue_type: { type: String, es_indexed: true }, + clinical_tumor_diagnosis: { type: String, es_indexed: false }, + histological_type: { type: String, set: deleteEmptyStrings, es_indexed: false }, + clinical_stage_grouping: { type: String, set: deleteEmptyStrings, es_indexed: false }, + site_of_sample_acquisition: { type: String, set: deleteEmptyStrings, es_indexed: false }, + tumor_histological_grade: { type: String, set: deleteEmptyStrings, es_indexed: false }, + licensing_required: { type: Boolean, es_indexed: true }, + distributor_part_number: { type: String, es_indexed: true }, + source_model_url: { type: String, es_indexed: true }, + source_sequence_url: { type: String, es_indexed: true }, + somatic_maf_url: { type: String, es_indexed: true }, + proteomics_url: { type: String, es_indexed: true }, + expanded: { type: Boolean, es_indexed: true }, + files: { type: [FilesSchema], es_indexed: true }, + variants: { type: [VariantExpression], es_indexed: true }, + variants_modified: { type: Boolean, es_indexed: false, default: false }, + genomic_variants: { type: [GenomicVariant], es_indexed: true }, + gene_metadata: { + type: GeneMetadata, + es_indexed: true, }, - { - es_extend: { - clinical_diagnosis: { - es_type: 'object', - es_value: (doc) => ({ - clinical_tumor_diagnosis: doc.clinical_tumor_diagnosis, - histological_type: doc.histological_type, - clinical_stage_grouping: doc.clinical_stage_grouping, - site_of_sample_acquisition: doc.site_of_sample_acquisition, - tumor_histological_grade: doc.tumor_histological_grade, - }), - }, - variants: { - es_type: 'nested', - es_value: (doc) => - doc.variants.map((variant) => ({ - assessment_type: variant.assessment_type, - expression_level: variant.expression_level, - category: variant.variant.category, - genes: variant.variant.genes, - name: variant.variant.name, - type: variant.variant.type, - })), - }, - genomic_variants: { - es_type: 'nested', - es_value: (doc) => - doc.genomic_variants.map((variant) => ({ - gene: variant.gene, - aa_change: variant.aa_change, - type: variant.type, - transcript_id: variant.transcript_id, - consequence_type: variant.consequence_type, - class: variant.class, - gene_biotype: variant.gene_biotype, - chromosome: variant.chromosome, - start_position: variant.start_position, - end_position: variant.end_position, - specific_change: variant.specific_change, - classification: variant.classification, - ensemble_id: variant.ensemble_id, - synonyms: variant.synonyms, - entrez_id: variant.entrez_id, - variant_id: variant.variant_id, - name: `${variant.gene} ${variant.aa_change}`, - })), - }, - gene_metadata: { - es_type: 'object', - es_value: (doc) => { - // Assemble list of genes from genomic_variants.gene and variants.variant.genes - const genomic_variant_genes = doc.genomic_variants.map((gv) => gv.gene); - const variant_genes = flatten(doc.variants.map((wrapper) => wrapper.variant.genes)); - const genes = uniq([...genomic_variant_genes, ...variant_genes]); - // As of #946, "Mutated Genes" are Research Somatic Variants (`genomic_variants` in the codebase) and Clinical Variants only - const clinical_variant_genes = flatten( - doc.variants - .filter((variant) => variant.variant && variant.variant.type === 'Clinical') - .map((wrapper) => wrapper.variant.genes), - ); - const mutated_genes = uniq([...genomic_variant_genes, ...clinical_variant_genes]); - - // Get counts of the 4 categories shown on search table - const genes_count = genes.length; - const mutated_genes_count = mutated_genes.length; - const genomic_variant_count = doc.genomic_variants.length; - const clinical_variant_count = doc.variants.filter( - (variant) => variant.variant && variant.variant.type === 'Clinical', - ).length; - const histopathological_variant_count = doc.variants.filter( - (variant) => variant.variant && variant.variant.type === 'Histopathological Biomarker', - ).length; - - const output = { - genes, - genes_count, - genomic_variant_count, - clinical_variant_count, - histopathological_variant_count, - mutated_genes, - mutated_genes_count, - }; - if (doc.gene_metadata) { - output.filename = doc.gene_metadata.filename; - output.import_data = doc.gene_metadata.import_date; - output.file_id = doc.gene_metadata.file_id; - } - return output; - }, - }, - // The following matched_models work is definitely a trick. You need to add populatedMatches as - // an array of models that should be included as matched_models before calling esIndex() - matched_models: { - es_type: 'nested', - es_value: (doc) => - (doc.populatedMatches || []).map((match) => ({ - name: match.name, - tissue_type: match.tissue_type, - })), - }, - has_matched_models: { - es_type: 'boolean', - es_value: (doc) => (doc.populatedMatches ? doc.populatedMatches.length >= 1 : false), - }, - matched_models_list: { - es_value: (doc) => - (doc.populatedMatches || []) - .concat([doc]) - .map((i) => i.name) - .join(','), - }, - - createdAt: { - es_type: 'date', - es_value: (doc) => doc.createdAt, - }, - updatedAt: { - es_type: 'date', - es_value: (doc) => doc.updatedAt, - }, - }, - timestamps: true, - collection: process.env.MONGO_COLLECTION, + matchedModels: { + type: mongoose.Schema.Types.ObjectId, + ref: 'MatchedModels', + es_indexed: false, }, -); + updateOldMatchesOnPublish: { + type: [String], + es_indexed: false, + }, + status: { + type: String, + enum: [modelStatus.unpublished, modelStatus.published, modelStatus.unpublishedChanges], + default: modelStatus.unpublished, + es_indexed: false, + }, + createdAt: { type: Date, default: Date.now, es_indexed: true }, + updatedAt: { type: Date, es_indexed: true }, + updatedBy: { type: String, es_indexed: false }, +}); export default mongoose.model('Model', ModelSchema); diff --git a/cms/src/services/elastic-search/common/client.js b/cms/src/services/elastic-search/common/client.js deleted file mode 100644 index c7b8c7ca..00000000 --- a/cms/src/services/elastic-search/common/client.js +++ /dev/null @@ -1,7 +0,0 @@ -import { Client } from '@elastic/elasticsearch'; -const eshost = `${process.env.ES_HOST || 'http://localhost'}:${process.env.ES_PORT || 9200}`; -const client = new Client({ - node: eshost, -}); - -export default client; diff --git a/cms/src/services/elastic-search/common/schemas/model.js b/cms/src/services/elastic-search/common/schemas/model.js deleted file mode 100644 index 026329ae..00000000 --- a/cms/src/services/elastic-search/common/schemas/model.js +++ /dev/null @@ -1,14 +0,0 @@ -import mongoose from 'mongoose'; -import mongooseElasticsearch from 'mongoose-elasticsearch-xp'; -import { ModelSchema } from '../../../../schemas/model.js'; -import elasticClient from '../client.js'; - -const index = process.env.ES_INDEX; - -ModelSchema.plugin(mongooseElasticsearch.v7, { - client: elasticClient, - index, - type: '_doc', -}); - -export const ModelES = mongoose.model('ModelES', ModelSchema); diff --git a/cms/src/services/elastic-search/publish.js b/cms/src/services/elastic-search/publish.js deleted file mode 100644 index dc7ac04e..00000000 --- a/cms/src/services/elastic-search/publish.js +++ /dev/null @@ -1,143 +0,0 @@ -import { ModelES } from './common/schemas/model.js'; -import Model from '../../schemas/model.js'; -import getPublishValidation from '../../validation/model.js'; -import { modelStatus } from '../../helpers/modelStatus.js'; -import MatchUtils from '../../helpers/matchedModels.js'; -import indexEsUpdate from './update.js'; -import { updateGeneSearchIndicies } from './genomicVariants.js'; - -import getLogger from '../../logger.js'; -const logger = getLogger('services/elastic-search/publish'); - -export const publishModel = async (filter, individualPublish = true) => { - await indexOneToES(filter); - await indexMatchedModelsToES(filter); - - // For individual publishes, update gene search indices immediately (if required) - if (individualPublish) { - const modelWithVariantChanges = await Model.findOne({ ...filter, variants_modified: true }); - if (modelWithVariantChanges) { - await updateGeneSearchIndicies(); - modelWithVariantChanges.variants_modified = false; - await modelWithVariantChanges.save(); - } - } -}; - -// For bulk publishes, update gene search indices after all models are published -export const bulkUpdateGeneSearchIndices = async modelNames => { - const modelsWithVariantChanges = await Model.find({ - name: { $in: modelNames }, - variants_modified: true, - }); - - if (modelsWithVariantChanges.length) { - await updateGeneSearchIndicies(); - // Reset the `variants_modified` flag to false now that gene search indices have been updated - await Model.updateMany( - { name: { $in: modelNames }, variants_modified: true }, - { $set: { variants_modified: false } }, - ); - } -}; - -export const indexOneToES = filter => { - return new Promise((resolve, reject) => { - ModelES.findOne(filter) - .populate('variants.variant') - .populate('matchedModels') - .exec(async (err, doc) => { - if (err) { - reject(err); - } - const validation = await getPublishValidation(); - // Validate doc against publish schema - // for "on-demand" publishing - validation - .validate(doc) - .then(async () => { - // Need to populate and filter the matched models - if (doc.matchedModels) { - const matchedModels = await ModelES.find({ - _id: { $in: doc.matchedModels.models || [] }, - }); - const matches = matchedModels.filter( - model => model.status !== modelStatus.unpublished && model.name !== doc.name, - ); - doc.populatedMatches = matches; - } - doc.esIndex((err, res) => { - if (err) { - reject(err); - } else { - indexEsUpdate() && - resolve({ - status: `Indexing successful with status: ${res.result}`, - }); - logger.audit({ model: doc.name }, 'publish model', 'Model Published to ES'); - } - }); - }) - .then( - async () => - await Model.updateOne({ name: doc.name }, { status: modelStatus.published }), - ) - .catch(err => reject(err)); - }); - }); -}; - -/** - * Provided a Model Name, this will find the matched model set for that model, and re-publish all other - * members of that matched set. - * This is needed since when a model in a set is published all members of that set need to have their - * matched model content updated to include the new published details. - * @param String updatedModelName is the Name of a model, all the other published members of its - * matchedModel set will be republished. - */ -export const indexMatchedModelsToES = async (filter, skipSelf = true) => { - const model = await ModelES.findOne(filter).populate('matchedModels'); - - // Get list of matched models if exists - const matchedModelIds = model.matchedModels ? model.matchedModels.models : []; - if (model.matchedModels && matchedModelIds.length <= 1) { - // if list of matched models exists but only includes itself, we can remove the whole matched model set. - logger.warn( - `Model ${model.name} is part of a small or empty matchedModel set of length ${ - matchedModelIds.length - }, deleting the set and the reference in the model.`, - ); - await MatchUtils.removeFromSet(model.name); - } - - if (model.updateOldMatchesOnPublish) { - // if we need to update old matches, we can do that here - matchedModelIds.push(...model.updateOldMatchesOnPublish); - } - - await updateMatchedModelsToES({ - _id: { $in: matchedModelIds.filter(id => id.toString() !== model._id.toString() || !skipSelf) }, - }); - - if (model.updateOldMatchesOnPublish) { - // Remove the updateMatchedModels list now that we've updated them. - await Model.updateOne(filter, { updateOldMatchesOnPublish: [] }); - } -}; - -export const updateMatchedModelsToES = async filter => { - const models = await ModelES.find(filter); - - // filter this list only to models that are published or published with changes - const modelsToPublish = models.filter(model => model.status !== modelStatus.unpublished); - - logger.debug( - { matchedModels: modelsToPublish.map(model => model.name) }, - 'Updating matched models', - ); - for (let model of modelsToPublish) { - // Publish this model to ensure it has matchedModel updates, unless skepSelf is true and this model is the one named in the method argument name - logger.debug({ model: model.name }, `Publishing model in order to update Matched Models.`); - await indexOneToES({ name: model.name }); - } -}; diff --git a/cms/src/services/elastic-search/update.js b/cms/src/services/elastic-search/update.js deleted file mode 100644 index 75fea3b2..00000000 --- a/cms/src/services/elastic-search/update.js +++ /dev/null @@ -1,23 +0,0 @@ -// @ts-check - -import elasticClient from './common/client.js'; -import getLogger from '../../logger.js'; -const logger = getLogger('services/elastic-search/update'); - -const index = process.env.ES_UPDATE_INDEX; - -const update = () => - elasticClient - .index({ - index, - type: index, - body: { - date: Date.now(), - }, - }) - .catch(error => - // Catch here as we do not want an error here to block execution of the app - logger.error(error, index, `Error creating a new update for index`), - ); - -export default update; diff --git a/cms/src/services/gdc-importer/VariantImporter.js b/cms/src/services/gdc-importer/VariantImporter.js index 67c5e9cd..d602d189 100644 --- a/cms/src/services/gdc-importer/VariantImporter.js +++ b/cms/src/services/gdc-importer/VariantImporter.js @@ -9,7 +9,10 @@ import { getBulkMafStatus, getCancerModelFilesFromMafFileData, } from './mafFiles.js'; -import { addGenomicVariantsFromMaf, getGdcImportErrorMessage } from '../../helpers/genomicVariants.js'; +import { + addGenomicVariantsFromMaf, + getGdcImportErrorMessage, +} from '../../helpers/genomicVariants.js'; import getLogger from '../../logger.js'; const logger = getLogger('services/gdc-importer/VariantImporter'); @@ -27,7 +30,7 @@ const ImportTypes = { individual: 'INDIVIDUAL', }; -const getTissueStatus = async modelName => { +const getTissueStatus = async (modelName) => { const model = await Model.findOne({ name: modelName }); if (!model) { @@ -73,7 +76,7 @@ const Import = ({ logger.info({ startTime, stopTime, modelName }, 'Genomic Variant Import complete.'); }; - const errorStop = errorData => { + const errorStop = (errorData) => { status = ImportStatus.error; error = errorData; stopTime = Date.now(); @@ -115,7 +118,7 @@ const Import = ({ tissueStatus, }); - const parseMaf = maf => { + const parseMaf = (maf) => { // clear all the weird comments const withoutComments = maf.replace(/#.+\n/g, ''); return tsv.parse(withoutComments); @@ -138,7 +141,7 @@ const Import = ({ { time: Date.now(), startTime, fileId, filename, modelName }, 'Beginning MAF file download...', ); - const mafFile = await downloadMaf({ filename, fileId, modelName }).catch(error => { + const mafFile = await downloadMaf({ filename, fileId, modelName }).catch((error) => { errorStop({ code: IMPORT_ERRORS.manualImportError, message: error.message, @@ -186,7 +189,7 @@ const Import = ({ }; }; -const VariantImporter = (function() { +const VariantImporter = (function () { let queue = []; let failed = []; let stopped = []; @@ -194,13 +197,13 @@ const VariantImporter = (function() { let running = false; const cleanLists = () => { - failed = failed.filter(i => i && i.getData && !i.getData().acknowledged); - stopped = stopped.filter(i => i && i.getData && !i.getData().acknowledged); - success = success.filter(i => i && i.getData && !i.getData().acknowledged); + failed = failed.filter((i) => i && i.getData && !i.getData().acknowledged); + stopped = stopped.filter((i) => i && i.getData && !i.getData().acknowledged); + success = success.filter((i) => i && i.getData && !i.getData().acknowledged); // queue should never have anything acknowledged (should move to failed/stopped/success) // clearing just in case - queue = queue.filter(i => i && i.getData && !i.getData().acknowledged); + queue = queue.filter((i) => i && i.getData && !i.getData().acknowledged); }; const emptyQueue = () => { @@ -311,7 +314,7 @@ const VariantImporter = (function() { } }; - const queueBulkImport = async models => { + const queueBulkImport = async (models) => { if (!Array.isArray(models) || models.length < 1) { logger.error( 'queueBulkImport failed due to bad input. `models` must be an array of model names.', @@ -326,12 +329,12 @@ const VariantImporter = (function() { } // Remove duplicate imports - stopBulkImport(models); + await stopBulkImport(models); acknowledgeBulk(models); // Filter out models that don't exist within HCMI db let noMatchingModel = []; - models = models.filter(async modelName => { + models = models.filter(async (modelName) => { let match = await Model.findOne({ name: modelName }); if (!match) { @@ -352,7 +355,7 @@ const VariantImporter = (function() { failed = [ ...failed, // Non-actionable errors (no match, not found in GDC, no MAFs) - ...noMatchingModel.map(modelName => + ...noMatchingModel.map((modelName) => Import({ modelName, status: ImportStatus.error, @@ -363,7 +366,7 @@ const VariantImporter = (function() { importType: ImportTypes.bulk, }), ), - ...modelsStatus[GDC_MODEL_STATES.modelNotFound].map(modelName => + ...modelsStatus[GDC_MODEL_STATES.modelNotFound].map((modelName) => Import({ modelName, status: ImportStatus.error, @@ -374,7 +377,7 @@ const VariantImporter = (function() { importType: ImportTypes.bulk, }), ), - ...modelsStatus[GDC_MODEL_STATES.noMafs].map(modelName => + ...modelsStatus[GDC_MODEL_STATES.noMafs].map((modelName) => Import({ modelName, status: ImportStatus.error, @@ -387,7 +390,7 @@ const VariantImporter = (function() { ), // Actionable errors (multiple ngcm, no ngcm) ...(await Promise.all( - modelsStatus[GDC_MODEL_STATES.multipleNgcm].map(async modelName => + modelsStatus[GDC_MODEL_STATES.multipleNgcm].map(async (modelName) => Import({ modelName, caseId: modelsFileData[modelName].caseId, @@ -404,7 +407,7 @@ const VariantImporter = (function() { ), )), ...(await Promise.all( - modelsStatus[GDC_MODEL_STATES.noNgcm].map(async modelName => + modelsStatus[GDC_MODEL_STATES.noNgcm].map(async (modelName) => Import({ modelName, caseId: modelsFileData[modelName].caseId, @@ -425,7 +428,7 @@ const VariantImporter = (function() { // Queue imports for conflict-free models (single NGCM, single NGCM+) queue = [ ...queue, - ...modelsStatus[GDC_MODEL_STATES.singleNgcm].map(modelName => { + ...modelsStatus[GDC_MODEL_STATES.singleNgcm].map((modelName) => { const fileData = getCancerModelFilesFromMafFileData(modelsFileData[modelName], true)[0]; return Import({ modelName, @@ -435,7 +438,7 @@ const VariantImporter = (function() { importType: ImportTypes.bulk, }); }), - ...modelsStatus[GDC_MODEL_STATES.singleNgcmPlusEngcm].map(modelName => { + ...modelsStatus[GDC_MODEL_STATES.singleNgcmPlusEngcm].map((modelName) => { const fileData = getCancerModelFilesFromMafFileData(modelsFileData[modelName], true)[0]; return Import({ modelName, @@ -455,19 +458,19 @@ const VariantImporter = (function() { return { success: true, startTime: Date.now() }; }; - const stopImport = async modelName => { + const stopImport = async (modelName) => { // In case we get into an invalid state with multiple imports for a given model name, // we'll use filter to get the whole list of them. - const targets = queue.filter(i => i && i.modelName === modelName); + const targets = queue.filter((i) => i && i.modelName === modelName); if (targets.length) { - targets.forEach(target => target.stop()); + targets.forEach((target) => target.stop()); stopped = [...stopped, ...targets]; - queue = queue.filter(i => i && i.modelName !== modelName); + queue = queue.filter((i) => i && i.modelName !== modelName); } cleanLists(); - return targets.map(target => target.getData()); + return targets.map((target) => target.getData()); }; const stopBulkImport = async (modelNames = []) => { @@ -475,73 +478,73 @@ const VariantImporter = (function() { let targets = []; if (modelNames.length) { - modelNames.forEach(modelName => { - targets = [...targets, ...queue.filter(i => i && i.modelName === modelName)]; + modelNames.forEach((modelName) => { + targets = [...targets, ...queue.filter((i) => i && i.modelName === modelName)]; }); } else { targets = queue; } if (targets.length) { - targets.forEach(target => target.stop()); + targets.forEach((target) => target.stop()); stopped = [...stopped, ...targets]; emptyQueue(); } cleanLists(); - return targets.map(target => target.getData()); + return targets.map((target) => target.getData()); }; const getStatus = () => { cleanLists(); return { - queue: queue.map(i => i.getData()), - failed: failed.map(i => i.getData()), - stopped: stopped.map(i => i.getData()), - success: success.map(i => i.getData()), + queue: queue.map((i) => i.getData()), + failed: failed.map((i) => i.getData()), + stopped: stopped.map((i) => i.getData()), + success: success.map((i) => i.getData()), running, }; }; - const acknowledge = modelName => { + const acknowledge = (modelName) => { const targets = [ - ...failed.filter(i => i && i.modelName === modelName), - ...stopped.filter(i => i && i.modelName === modelName), - ...success.filter(i => i && i.modelName === modelName), + ...failed.filter((i) => i && i.modelName === modelName), + ...stopped.filter((i) => i && i.modelName === modelName), + ...success.filter((i) => i && i.modelName === modelName), ]; if (targets.length) { - targets.forEach(target => target.acknowledge()); + targets.forEach((target) => target.acknowledge()); } cleanLists(); - return targets.map(target => target.getData()); + return targets.map((target) => target.getData()); }; - const acknowledgeBulk = modelNames => { + const acknowledgeBulk = (modelNames) => { let targets = []; - modelNames.forEach(modelName => { + modelNames.forEach((modelName) => { targets = [ ...targets, - ...failed.filter(i => i && i.modelName === modelName), - ...stopped.filter(i => i && i.modelName === modelName), - ...success.filter(i => i && i.modelName === modelName), + ...failed.filter((i) => i && i.modelName === modelName), + ...stopped.filter((i) => i && i.modelName === modelName), + ...success.filter((i) => i && i.modelName === modelName), ]; }); if (targets.length) { - targets.forEach(target => target.acknowledge()); + targets.forEach((target) => target.acknowledge()); } cleanLists(); - return targets.map(target => target.getData()); + return targets.map((target) => target.getData()); }; const resolveConflict = (modelName, fileId, filename) => { - const targetIndex = failed.findIndex(i => i.modelName === modelName); + const targetIndex = failed.findIndex((i) => i.modelName === modelName); if (targetIndex < 0) { return { diff --git a/cms/src/services/gdc-importer/mafFiles.js b/cms/src/services/gdc-importer/mafFiles.js index 872a42e3..9e78f931 100644 --- a/cms/src/services/gdc-importer/mafFiles.js +++ b/cms/src/services/gdc-importer/mafFiles.js @@ -5,7 +5,8 @@ import { PassThrough } from 'stream'; import decompress from 'decompress'; import zlib from 'zlib'; import _ from 'lodash'; -const { get, flattenDeep, intersection, isEmpty } = _; + +import getLogger from '../../logger.js'; import { GDC_MODEL_STATES, @@ -14,10 +15,10 @@ import { FETCH_MODEL_FILE_DATA_QUERY, } from './gdcConstants.js'; -import getLogger from '../../logger.js'; +const { get, flattenDeep, intersection, isEmpty } = _; const logger = getLogger('services/gdc-importer/mafFiles'); -export const fetchModelFileData = async modelNames => { +export const fetchModelFileData = async (modelNames) => { if (!Array.isArray(modelNames) || !modelNames.length) { logger.error('fetchModelFileData failed due to invalid input. `modelNames` must be an array.', { modelNames, @@ -76,15 +77,15 @@ export const fetchModelFileData = async modelNames => { const caseId = get(cases[i], 'node.case_id'); const modelName = get(cases[i], 'node.submitter_id'); - const samples = get(cases[i], 'node.samples.hits.edges', []).map(sampleEdge => { + const samples = get(cases[i], 'node.samples.hits.edges', []).map((sampleEdge) => { const sampleType = get(sampleEdge, 'node.sample_type'); const tissueType = get(sampleEdge, 'node.tissue_type'); const tumorDescriptor = get(sampleEdge, 'node.tumor_descriptor'); // aliquots will be an array of two ids const aliquots = flattenDeep( - get(sampleEdge, 'node.portions.hits.edges', []).map(portionEdge => - get(portionEdge, 'node.analytes.hits.edges', []).map(analyteEdge => - get(analyteEdge, 'node.aliquots.hits.edges', []).map(aliquot => + get(sampleEdge, 'node.portions.hits.edges', []).map((portionEdge) => + get(portionEdge, 'node.analytes.hits.edges', []).map((analyteEdge) => + get(analyteEdge, 'node.aliquots.hits.edges', []).map((aliquot) => get(aliquot, 'node.aliquot_id'), ), ), @@ -96,27 +97,27 @@ export const fetchModelFileData = async modelNames => { logger.debug({ caseId, samples }, `Case samples found for model ${modelName}`); const files = get(data, 'files.hits.edges', []) - .filter(fileEdge => { + .filter((fileEdge) => { const entitySubmitterIds = get(fileEdge, 'node.associated_entities.hits.edges', []).map( - entityEdge => get(entityEdge, 'node.entity_submitter_id'), + (entityEdge) => get(entityEdge, 'node.entity_submitter_id'), ); - return entitySubmitterIds.some(submitterId => submitterId.includes(modelName)); + return entitySubmitterIds.some((submitterId) => submitterId.includes(modelName)); }) - .map(fileEdge => { + .map((fileEdge) => { const fileId = get(fileEdge, 'node.file_id'); const filename = get(fileEdge, 'node.file_name'); const entityIds = get(fileEdge, 'node.associated_entities.hits.edges', []).map( - entityEdge => get(entityEdge, 'node.entity_id'), + (entityEdge) => get(entityEdge, 'node.entity_id'), ); const entities = entityIds - .filter(entity => { - return samples.some(sample => sample.aliquots.includes(entity)); + .filter((entity) => { + return samples.some((sample) => sample.aliquots.includes(entity)); }) - .map(entity => { - const matchingSample = samples.find(sample => sample.aliquots.includes(entity)); + .map((entity) => { + const matchingSample = samples.find((sample) => sample.aliquots.includes(entity)); const matchingEntity = get(fileEdge, 'node.associated_entities.hits.edges', []).find( - x => x.node.entity_id === entity, + (x) => x.node.entity_id === entity, ); return { entityId: entity, @@ -143,7 +144,7 @@ export const fetchModelFileData = async modelNames => { return results; }; -export const getMafStatus = mafFileData => { +export const getMafStatus = (mafFileData) => { if (!mafFileData.success) { // Model not found in GDC return GDC_MODEL_STATES.modelNotFound; @@ -169,7 +170,7 @@ export const getMafStatus = mafFileData => { (totals, currentModelFile) => { let currentNgcmCount = 0; let currentEngcmCount = 0; - currentModelFile.entities.forEach(entity => { + currentModelFile.entities.forEach((entity) => { switch (entity.sampleType) { case GDC_CANCER_MODEL_SAMPLE_TYPES.NGCM: currentNgcmCount++; @@ -206,7 +207,7 @@ export const getMafStatus = mafFileData => { } }; -export const getBulkMafStatus = bulkMafFileData => { +export const getBulkMafStatus = (bulkMafFileData) => { const models = Object.keys(bulkMafFileData); const results = Object.values(GDC_MODEL_STATES).reduce((o, key) => ({ ...o, [key]: [] }), {}); @@ -276,7 +277,13 @@ const filterMafFilesBySampleTypes = (files, sampleTypes) => { } return files.filter( - file => !isEmpty(intersection(file.entities.map(entity => entity.sampleType), sampleTypes)), + (file) => + !isEmpty( + intersection( + file.entities.map((entity) => entity.sampleType), + sampleTypes, + ), + ), ); }; @@ -306,15 +313,15 @@ export const downloadMaf = async ({ filename, fileId, modelName }) => { const streamToBuffer = new PassThrough(); const bufs = []; - streamToBuffer.on('data', data => { + streamToBuffer.on('data', (data) => { bufs.push(data); }); streamToBuffer.on('end', () => { decompress(Buffer.concat(bufs), { - filter: file => file.path.includes(filename), + filter: (file) => file.path.includes(filename), strip: 1, }) - .then(files => { + .then((files) => { try { const maf = zlib.gunzipSync(files[0].data).toString('utf8'); @@ -327,7 +334,7 @@ export const downloadMaf = async ({ filename, fileId, modelName }) => { reject(error); } }) - .catch(error => { + .catch((error) => { logger.error( { error, filename, fileId, modelName }, 'Failure decompressing file from GDC', diff --git a/cms/src/services/publish/Publisher.js b/cms/src/services/publish/Publisher.js index e62622f7..0871f400 100644 --- a/cms/src/services/publish/Publisher.js +++ b/cms/src/services/publish/Publisher.js @@ -3,7 +3,7 @@ import getPublishValidation from '../../validation/model.js'; import { runYupValidatorFailSlow } from '../../helpers/index.js'; import { PUBLISH_ERRORS } from './constants.js'; import { getPublishErrorMessage } from './helpers.js'; -import { publishModel, bulkUpdateGeneSearchIndices } from '../elastic-search/publish.js'; +import { publishModel, bulkUpdateGeneSearchIndices } from '../search-client/publish.js'; import getLogger from '../../logger.js'; const logger = getLogger('services/publish/Publisher'); @@ -145,7 +145,7 @@ const Publisher = (function () { } // Remove duplicate publish tasks - stopPublish(modelName); + await stopPublish(modelName); acknowledge(modelName); try { @@ -157,7 +157,7 @@ const Publisher = (function () { name: modelName, }) .populate('variants.variant') - .then((model) => runYupValidatorFailSlow(validation, model)) + .then(async (model) => await runYupValidatorFailSlow(validation, model)) .then((results) => { if (results[0].success) { // Create new publish task @@ -228,7 +228,7 @@ const Publisher = (function () { } // Remove duplicate imports - stopBulkPublish(models); + await stopBulkPublish(models); acknowledgeBulk(models); // Filter out models that don't exist within HCMI db @@ -410,9 +410,9 @@ const Publisher = (function () { running = false; }; - const start = () => { + const start = async () => { running = true; - run(); + await run(); }; const run = async () => { @@ -446,7 +446,7 @@ const Publisher = (function () { } if (running && queue.length > 0) { - run(); + await run(); } else { // Update gene search indices after bulk publish completes await updateBulkGeneSearchIndicies(); diff --git a/cms/src/services/s3/s3.js b/cms/src/services/s3/s3.js index 3da666fc..ffaf72e5 100644 --- a/cms/src/services/s3/s3.js +++ b/cms/src/services/s3/s3.js @@ -31,7 +31,7 @@ const uploadToS3 = async (fileName, fileStream, modelName) => { if (error) { reject({ error, fileName, modelName }); } else { - logger.audit( + logger.info( { Key, Bucket: S3_BUCKET, response: data }, 's3 upload', `Successfully uploaded object to S3`, @@ -56,7 +56,7 @@ const deleteFromS3 = async (id) => { msg: `image with id ${id} not found`, }; } else { - logger.audit(params, 's3 delete', 'Successfully deleted image from S3'); + logger.info(params, 's3 delete', 'Successfully deleted image from S3'); return { code: 200, msg: `image with id ${id} deleted`, diff --git a/cms/src/services/search-client/client.js b/cms/src/services/search-client/client.js new file mode 100644 index 00000000..be42ec9a --- /dev/null +++ b/cms/src/services/search-client/client.js @@ -0,0 +1,30 @@ +import { Client } from '@opensearch-project/opensearch'; +import { AwsSigv4Signer } from '@opensearch-project/opensearch/aws'; +import { defaultProvider } from '@aws-sdk/credential-provider-node'; + +const getClient = (pm2config) => { + const node = process.env.ES_URL || pm2config?.ES_URL || 'http://localhost:9200'; + const authType = process.env.SEARCH_ENGINE_AUTH_TYPE || pm2config?.SEARCH_ENGINE_AUTH_TYPE || ''; + const region = + process.env.SEARCH_ENGINE_AUTH_REGION || pm2config?.SEARCH_ENGINE_AUTH_REGION || ''; + const service = + process.env.SEARCH_ENGINE_AUTH_SERVICE || pm2config?.SEARCH_ENGINE_AUTH_SERVICE || ''; + const username = process.env.ES_USER || pm2config?.ES_USER || ''; + const password = process.env.ES_PASS || pm2config?.ES_PASS || ''; + + return authType === 'aws' + ? new Client({ + ...AwsSigv4Signer({ + region, + service, + getCredentials: () => defaultProvider()(), + }), + node, + }) + : new Client({ + node, + auth: username ? { username, password } : undefined, + }); +}; + +export default getClient; diff --git a/cms/src/services/elastic-search/genomicVariants.js b/cms/src/services/search-client/genomicVariants.js similarity index 68% rename from cms/src/services/elastic-search/genomicVariants.js rename to cms/src/services/search-client/genomicVariants.js index b09594b3..3f25bb7d 100644 --- a/cms/src/services/elastic-search/genomicVariants.js +++ b/cms/src/services/search-client/genomicVariants.js @@ -1,27 +1,32 @@ -import esClient from './common/client.js'; - +// @ts-check import _ from 'lodash'; -const { get, flatten, uniq } = _; import getLogger from '../../logger.js'; -const logger = getLogger('services/elastic-search/genomicVariants'); +import pm2 from '../../pm2.js'; + +import getClient from './client.js'; + +const { get, flatten, uniq } = _; + +const logger = getLogger('services/search-client/genomicVariants'); const MODEL_INDEX = process.env.ES_INDEX; const GENES_INDEX = 'genes'; const VARIANTS_INDEX = 'genomic_variants'; -const getAllIndexedDocs = async index => { +const getAllIndexedDocs = async (index) => { let output = []; let startFrom = 0; let totalHits = 1; const requestSize = 100; + const searchClient = getClient(pm2); while (totalHits > startFrom) { - const esResponse = await esClient.search({ + const esResponse = await searchClient.search({ index: index, size: requestSize, from: startFrom, }); - totalHits = esResponse.body.hits.total.value; + totalHits = esResponse?.body?.hits?.total?.value; output = output.concat(get(esResponse, 'body.hits.hits', [])); startFrom += requestSize; @@ -35,38 +40,41 @@ const getAllIndexedDocs = async index => { // can't be sure what is published for a model in draft form (changes not yet published) // Additionally, to know which genes to remove we have to compare this list to the currently published list in the genes index on ES -const updateVariantIndex = async desiredVariants => { +const updateVariantIndex = async (desiredVariants) => { + const searchClient = getClient(pm2); // TEMP: Remove all usage of variantIdMafOnly and just use variant_id once the gene reference is reimplemented - Jon Eubank 2020-09 - const variantIdMafOnly = originalID => `${originalID}_MAF`; + const variantIdMafOnly = (originalID) => `${originalID}_MAF`; logger.debug({ desiredVariants }, 'List of Variants that should be published'); - const variantIds = desiredVariants.map(v => variantIdMafOnly(v.variant.variant_id)); + const variantIds = desiredVariants.map((v) => variantIdMafOnly(v.variant.variant_id)); // 1. get list of variants published in ES const variantsResponse = await getAllIndexedDocs(VARIANTS_INDEX); - const publishedVariants = variantsResponse.map(variant => variant._id); + const publishedVariants = variantsResponse.map((variant) => variant._id); logger.debug({ publishedVariants }, 'List of Variants currently published'); // 2. find list of variants to unpublish and genes to publish - const removeVariants = publishedVariants.filter(publishedID => !variantIds.includes(publishedID)); + const removeVariants = publishedVariants.filter( + (publishedID) => !variantIds.includes(publishedID), + ); logger.debug({ removeVariants }, 'Variants to unpublish'); // TEMP: Remove the second clause, where we filter out Unknown named variants, when we re-implement the gene reference - Jon Eubank 2020-09 const addVariants = desiredVariants.filter( - v => + (v) => !publishedVariants.includes(variantIdMafOnly(v.variant.variant_id)) && !v.variant.name.includes('Unknown'), ); logger.debug({ addVariants }, 'Variants to publish'); // 3. do the publish/unpublish operations - const deleteRequests = removeVariants.map(variant => ({ + const deleteRequests = removeVariants.map((variant) => ({ delete: { _index: VARIANTS_INDEX, _id: variant }, })); const addRequests = flatten( - addVariants.map(v => { + addVariants.map((v) => { const doc = { name: v.variant.name, transcript_id: v.variant.transcript_id, @@ -86,7 +94,7 @@ const updateVariantIndex = async desiredVariants => { ); if (deleteRequests.length || addRequests.length) { - await esClient.bulk({ + await searchClient.bulk({ body: [...deleteRequests, ...addRequests], }); } else { @@ -94,32 +102,35 @@ const updateVariantIndex = async desiredVariants => { } }; -const updateGeneIndex = async desiredGenes => { - const desiredGeneIds = desiredGenes.map(gene => gene.symbol); +const updateGeneIndex = async (desiredGenes) => { + const searchClient = getClient(pm2); + const desiredGeneIds = desiredGenes.map((gene) => gene.symbol); logger.debug({ desiredGenes: desiredGeneIds }, 'List of Genes that should be published'); // 1. get list of genes published in ES const genesResponse = await getAllIndexedDocs(GENES_INDEX); - const publishedGenes = genesResponse.map(i => i._id); + const publishedGenes = genesResponse.map((i) => i._id); logger.debug({ publishedGenes }, 'List of Genes currently published'); // 2. find list of genes to unpublish and genes to publish - const removeGenes = uniq(publishedGenes.filter(gene => !desiredGeneIds.includes(gene))); + const removeGenes = uniq(publishedGenes.filter((gene) => !desiredGeneIds.includes(gene))); logger.debug({ removeGenes }, 'Genes to unpublish'); const addGenes = uniq( - desiredGenes.filter(gene => !publishedGenes.includes(gene.symbol)), + desiredGenes.filter((gene) => !publishedGenes.includes(gene.symbol)), 'symbol', ); logger.debug({ addGenes }, 'Genes to publish'); // 3. do the publish/unpublish operations - const deleteRequests = removeGenes.map(gene => ({ delete: { _index: GENES_INDEX, _id: gene } })); + const deleteRequests = removeGenes.map((gene) => ({ + delete: { _index: GENES_INDEX, _id: gene }, + })); const addRequests = flatten( - desiredGenes.map(gene => { + desiredGenes.map((gene) => { return [ { index: { @@ -133,7 +144,7 @@ const updateGeneIndex = async desiredGenes => { ); if (deleteRequests.length || addRequests.length) { - await esClient.bulk({ + await searchClient.bulk({ body: [...deleteRequests, ...addRequests], }); } else { @@ -142,17 +153,18 @@ const updateGeneIndex = async desiredGenes => { }; export const updateGeneSearchIndicies = async () => { + const searchClient = getClient(pm2); // This method reads from the es indices, and is prone to errors if we read it before updates have been indexed // so first, we refresh the model index :) - await esClient.indices.refresh({ index: MODEL_INDEX }); + await searchClient.indices.refresh({ index: MODEL_INDEX }); // 1. get models from ES const publishedModels = await getAllIndexedDocs(MODEL_INDEX); // 1a. collect set of genes and variants from those model variants const desiredGenomicVariants = flatten( - publishedModels.map(model => { - return (model._source.genomic_variants || []).map(variant => ({ + publishedModels.map((model) => { + return (model?._source?.genomic_variants || []).map((variant) => ({ variant: { transcript_id: variant.transcript_id, variant_id: variant.variant_id, @@ -167,17 +179,21 @@ export const updateGeneSearchIndicies = async () => { ); const clinicalVariantGenes = flatten( - publishedModels.map(model => flatten(model._source.variants.map(variant => variant.genes))), + publishedModels?.map((model) => + flatten(model?._source?.variants?.map((variant) => variant.genes) || []), + ), ); logger.debug({ clinicalVariantGenes }, 'Genes found in published model variants'); // Filter the list of clinical variants to only have the names missing from the genomic variants list - const desiredGeneSymbols = desiredGenomicVariants.map(i => i.gene.symbol); + const desiredGeneSymbols = desiredGenomicVariants.map((i) => i.gene.symbol); const additionalGenesFromClinical = clinicalVariantGenes - .filter(clinicalGene => !desiredGeneSymbols.includes(clinicalGene)) - .map(gene => ({ symbol: gene })); + .filter((clinicalGene) => !desiredGeneSymbols.includes(clinicalGene)) + .map((gene) => ({ symbol: gene })); - const desiredGenes = desiredGenomicVariants.map(i => i.gene).concat(additionalGenesFromClinical); + const desiredGenes = desiredGenomicVariants + .map((i) => i.gene) + .concat(additionalGenesFromClinical); await updateGeneIndex(desiredGenes); await updateVariantIndex(desiredGenomicVariants); diff --git a/cms/src/services/search-client/indexLastUpdated.js b/cms/src/services/search-client/indexLastUpdated.js new file mode 100644 index 00000000..b5defb31 --- /dev/null +++ b/cms/src/services/search-client/indexLastUpdated.js @@ -0,0 +1,26 @@ +// @ts-check +import pm2 from '../../pm2.js'; +import getLogger from '../../logger.js'; + +import getClient from './client.js'; + +const logger = getLogger('services/search-client/update'); + +const index = process.env.ES_UPDATE_INDEX || 'hcmi-update'; + +const indexLastUpdated = async () => { + const searchClient = getClient(pm2); + return searchClient + .index({ + index, + body: { + date: Date.now(), + }, + }) + .catch((error) => + // Catch here as we do not want an error here to block execution of the app + logger.error(error, index, `Error creating a new update for index`), + ); +}; + +export default indexLastUpdated; diff --git a/cms/src/services/search-client/indexModel.js b/cms/src/services/search-client/indexModel.js new file mode 100644 index 00000000..190151c9 --- /dev/null +++ b/cms/src/services/search-client/indexModel.js @@ -0,0 +1,27 @@ +// @ts-check +import getLogger from '../../logger.js'; +import pm2 from '../../pm2.js'; + +import getClient from './client.js'; + +const logger = getLogger('services/search-client/update'); + +const index = process.env.ES_INDEX || 'hcmi'; + +const indexModel = async (id, model) => { + const searchClient = getClient(pm2); + return searchClient + .index({ + index, + id, + body: { + ...model, + }, + }) + .catch((error) => { + logger.error(error, index, `Error indexing Model data`); + throw error; + }); +}; + +export default indexModel; diff --git a/cms/src/services/search-client/publish.js b/cms/src/services/search-client/publish.js new file mode 100644 index 00000000..32b0f480 --- /dev/null +++ b/cms/src/services/search-client/publish.js @@ -0,0 +1,307 @@ +import _ from 'lodash'; +import mongoose from 'mongoose'; + +import Model from '../../schemas/model.js'; +import getPublishValidation from '../../validation/model.js'; +import { modelStatus } from '../../helpers/modelStatus.js'; +import MatchUtils from '../../helpers/matchedModels.js'; +import getLogger from '../../logger.js'; + +import indexLastUpdated from './indexLastUpdated.js'; +import indexModel from './indexModel.js'; +import { updateGeneSearchIndicies } from './genomicVariants.js'; + +const logger = getLogger('services/search-client/publish'); + +/** + * Removes Mongoose specific keys & values to prepare data for Search indexing + */ +const cleanMongoDoc = (doc) => { + const mongoKeys = ['_id', '__v']; + // Remove keys from base Document object + let cleanedDoc = _.omit(doc, mongoKeys); + for (const key in cleanedDoc) { + // Review if nested values also need Mongoose keys removed + const value = cleanedDoc[key]; + if (value && typeof value === 'object') { + if (Array.isArray(value)) { + const firstEntry = value[0]; + // Remove nested keys when property is an array of objects + const cleanedVals = + firstEntry && typeof firstEntry === 'object' + ? value.map((val) => { + const cleanedValue = _.omit(val, mongoKeys); + return cleanedValue; + }) + : value; + cleanedDoc[key] = cleanedVals; + } else if (value instanceof mongoose.Types.ObjectId) { + // Parse Mongoose ObjectId Objects to a plain string + cleanedDoc[key] = value.toString(); + } else if (!(value instanceof Date)) { + // Remove keys when value is an object, ignoring Dates + const cleanedValue = _.omit(value, mongoKeys); + cleanedDoc[key] = cleanedValue; + } + } + } + return cleanedDoc; +}; + +/** + * Collects gene names and counts + */ +const getGeneMetadata = async (doc) => { + // Assemble list of genes from genomic_variants.gene and variants.variant.genes + const genomic_variant_genes = doc.genomic_variants.map((gv) => gv.gene); + const variant_genes = _.flatten(doc.variants.map((wrapper) => wrapper.variant.genes)); + const genes = _.uniq([...genomic_variant_genes, ...variant_genes]); + // "Mutated Genes" are Research Somatic Variants (`genomic_variants` in the codebase) and Clinical Variants only + const clinical_variant_genes = _.flatten( + doc.variants + .filter((variant) => variant.variant && variant.variant.type === 'Clinical') + .map((wrapper) => wrapper.variant.genes), + ); + const mutated_genes = _.uniq([...genomic_variant_genes, ...clinical_variant_genes]); + + // Get counts of the 4 categories shown on search table + const genes_count = genes.length; + const mutated_genes_count = mutated_genes.length; + const genomic_variant_count = doc.genomic_variants.length; + const clinical_variant_count = doc.variants.filter( + (variant) => variant.variant && variant.variant.type === 'Clinical', + ).length; + const histopathological_variant_count = doc.variants.filter( + (variant) => variant.variant && variant.variant.type === 'Histopathological Biomarker', + ).length; + + const filename = doc.gene_metadata?.filename; + const import_data = doc.gene_metadata?.import_date; + const file_id = doc.gene_metadata?.file_id; + + const output = { + clinical_variant_count, + file_id, + filename, + genes, + genes_count, + genomic_variant_count, + histopathological_variant_count, + import_data, + mutated_genes, + mutated_genes_count, + }; + + return output; +}; + +/** + * Returns an array containing names and tissue types of related Matched Models + */ +const getMatchedModels = async (modelRecord) => { + if (modelRecord.matchedModels) { + const matchedModels = await Model.find({ + _id: { $in: modelRecord.matchedModels.models || [] }, + }); + const matches = matchedModels + .filter( + (model) => model.status !== modelStatus.unpublished && model.name !== modelRecord.name, + ) + .map((record) => { + const { name, tissue_type } = record; + return { name, tissue_type }; + }); + + return matches; + } + return undefined; +}; + +/** + * Aggregates Model metadata and formats Model document for Search client indexing + * Ports logic previously found in schemas/model es_extends + */ +const formatModelToDocument = async (doc) => { + const modelRecord = { ...doc.toObject(), updatedAt: new Date() }; + + const clinical_diagnosis = { + clinical_tumor_diagnosis: modelRecord.clinical_tumor_diagnosis, + histological_type: modelRecord.histological_type, + clinical_stage_grouping: modelRecord.clinical_stage_grouping, + site_of_sample_acquisition: modelRecord.site_of_sample_acquisition, + tumor_histological_grade: modelRecord.tumor_histological_grade, + }; + + const variants = modelRecord.variants.map((variant) => ({ + assessment_type: variant.assessment_type, + expression_level: variant.expression_level, + category: variant.variant.category, + genes: variant.variant.genes, + name: variant.variant.name, + type: variant.variant.type, + })); + + const genomic_variants = modelRecord.genomic_variants.map((variant) => ({ + gene: variant.gene, + aa_change: variant.aa_change, + type: variant.type, + transcript_id: variant.transcript_id, + consequence_type: variant.consequence_type, + class: variant.class, + gene_biotype: variant.gene_biotype, + chromosome: variant.chromosome, + start_position: variant.start_position, + end_position: variant.end_position, + specific_change: variant.specific_change, + classification: variant.classification, + ensemble_id: variant.ensemble_id, + synonyms: variant.synonyms, + entrez_id: variant.entrez_id, + variant_id: variant.variant_id, + name: `${variant.gene} ${variant.aa_change}`, + })); + + const gene_metadata = await getGeneMetadata(modelRecord); + + const matched_models = await getMatchedModels(modelRecord); + + const has_matched_models = !!matched_models; + const matched_models_list = + matched_models + ?.concat([modelRecord]) + .map((i) => i.name) + .join(',') || ''; + + const mappedRecord = { + ...modelRecord, + clinical_diagnosis, + variants, + genomic_variants, + gene_metadata, + matched_models, + has_matched_models, + matched_models_list, + }; + + const cleanedDoc = cleanMongoDoc(mappedRecord); + return cleanedDoc; +}; + +export const publishModel = async (filter, individualPublish = true) => { + await indexOneToES(filter); + await indexMatchedModelsToES(filter); + + // For individual publishes, update gene search indices immediately (if required) + if (individualPublish) { + const modelWithVariantChanges = await Model.findOne({ ...filter, variants_modified: true }); + if (modelWithVariantChanges) { + await updateGeneSearchIndicies(); + modelWithVariantChanges.variants_modified = false; + await modelWithVariantChanges.save(); + } + } +}; + +// For bulk publishes, update gene search indices after all models are published +export const bulkUpdateGeneSearchIndices = async (modelNames) => { + const modelsWithVariantChanges = await Model.find({ + name: { $in: modelNames }, + variants_modified: true, + }); + + if (modelsWithVariantChanges.length) { + await updateGeneSearchIndicies(); + // Reset the `variants_modified` flag to false now that gene search indices have been updated + await Model.updateMany( + { name: { $in: modelNames }, variants_modified: true }, + { $set: { variants_modified: false } }, + ); + } +}; + +export const indexOneToES = async (filter) => { + try { + const validation = await getPublishValidation(); + const doc = await Model.findOne(filter) + .populate('variants.variant') + .populate('matchedModels') + .exec(); + + // Validate doc against publish schema for "on-demand" publishing + await validation.validate(doc); + // Need to populate and filter the matched models, and format data for indexing + const data = await formatModelToDocument(doc); + + // Index model into ElasticSearch + await indexModel(doc._id, data); + await indexLastUpdated(); + + const res = await Model.updateOne( + { name: doc.name }, + { status: modelStatus.published, updatedAt: data.updatedAt }, + ); + + logger.info({ model: doc.name }, 'publish model', 'Model Published to ES'); + return { + status: `Indexing successful with status: ${res.result}`, + }; + } catch (err) { + logger.error('Error at indexOneToES', err); + throw err; + } +}; + +/** + * Provided a Model Name, this will find the matched model set for that model, and re-publish all other + * members of that matched set. + * This is needed since when a model in a set is published all members of that set need to have their + * matched model content updated to include the new published details. + * @param String updatedModelName is the Name of a model, all the other published members of its + * matchedModel set will be republished. + */ +export const indexMatchedModelsToES = async (filter, skipSelf = true) => { + const model = await Model.findOne(filter).populate('matchedModels'); + + // Get list of matched models if exists + const matchedModelIds = model.matchedModels ? model.matchedModels.models : []; + if (model.matchedModels && matchedModelIds.length <= 1) { + // if list of matched models exists but only includes itself, we can remove the whole matched model set. + logger.warn( + `Model ${model.name} is part of a small or empty matchedModel set of length ${matchedModelIds.length}, deleting the set and the reference in the model.`, + ); + await MatchUtils.removeFromSet(model.name); + } + + if (model.updateOldMatchesOnPublish) { + // if we need to update old matches, we can do that here + matchedModelIds.push(...model.updateOldMatchesOnPublish); + } + + await updateMatchedModelsToES({ + _id: { + $in: matchedModelIds.filter((id) => id.toString() !== model._id.toString() || !skipSelf), + }, + }); + + if (model.updateOldMatchesOnPublish) { + // Remove the updateMatchedModels list now that we've updated them. + await Model.updateOne(filter, { updateOldMatchesOnPublish: [] }); + } +}; + +export const updateMatchedModelsToES = async (filter) => { + const models = await Model.find(filter); + + // filter this list only to models that are published or published with changes + const modelsToPublish = models.filter((model) => model.status !== modelStatus.unpublished); + + logger.debug( + { matchedModels: modelsToPublish.map((model) => model.name) }, + 'Updating matched models', + ); + for (let model of modelsToPublish) { + // Publish this model to ensure it has matchedModel updates, unless skepSelf is true and this model is the one named in the method argument name + logger.debug({ model: model.name }, `Publishing model in order to update Matched Models.`); + await indexOneToES({ name: model.name }); + } +}; diff --git a/cms/src/services/elastic-search/unpublish.js b/cms/src/services/search-client/unpublish.js similarity index 55% rename from cms/src/services/elastic-search/unpublish.js rename to cms/src/services/search-client/unpublish.js index 22f667c7..792206b6 100644 --- a/cms/src/services/elastic-search/unpublish.js +++ b/cms/src/services/search-client/unpublish.js @@ -1,28 +1,31 @@ // @ts-check -import elasticClient from './common/client.js'; -import indexEsUpdate from './update.js'; import Model from '../../schemas/model.js'; -import { indexMatchedModelsToES } from './publish.js'; import { modelStatus } from '../../helpers/modelStatus.js'; +import getLogger from '../../logger.js'; +import pm2 from '../../pm2.js'; + +import getClient from './client.js'; +import indexLastUpdated from './indexLastUpdated.js'; +import { indexMatchedModelsToES } from './publish.js'; import { updateGeneSearchIndicies } from './genomicVariants.js'; -import getLogger from '../../logger.js'; -const logger = getLogger('services/elastic-search/unpublish'); +const logger = getLogger('services/search-client/unpublish'); const index = process.env.ES_INDEX; -export const unpublishModel = async name => { +export const unpublishModel = async (name) => { await unpublishOneFromES(name); await indexMatchedModelsToES({ name }); - updateGeneSearchIndicies(); + await updateGeneSearchIndicies(); }; -export const unpublishOneFromES = async name => { +export const unpublishOneFromES = async (name) => { // Not waiting for update promise to // resolve as this is just bookkeeping - indexEsUpdate(); - await elasticClient.deleteByQuery({ + await indexLastUpdated(); + const searchClient = getClient(pm2); + await searchClient.deleteByQuery({ index, body: { query: { @@ -36,14 +39,15 @@ export const unpublishOneFromES = async name => { }, { status: modelStatus.unpublished }, ); - logger.audit({ model: name }, 'unpublish model', 'Model Unpublished from ES'); + logger.info({ model: name }, 'unpublish model', 'Model Unpublished from ES'); }; -export const unpublishManyFromES = nameArr => { +export const unpublishManyFromES = async (nameArr) => { // Not waiting for update promise to // resolve as this is just bookkeeping - indexEsUpdate(); - return elasticClient.deleteByQuery({ + await indexLastUpdated(); + const searchClient = getClient(pm2); + return await searchClient.deleteByQuery({ index, body: { query: { diff --git a/cms/src/validation/getPublishSchema.js b/cms/src/validation/getPublishSchema.js index ad8f1fb8..5ac47a4c 100644 --- a/cms/src/validation/getPublishSchema.js +++ b/cms/src/validation/getPublishSchema.js @@ -1,5 +1,4 @@ import * as yup from 'yup'; -import moment from 'moment'; import { arrItemIsOneOf, nameRegex, @@ -40,12 +39,22 @@ const getPublishSchema = async (excludedNames, dictionary) => { .notOneOf(excludedNames, 'This model already exists'), expanded: boolean().required('This is a required field'), type: string().oneOf(modelTypeOptions), - growth_rate: number().integer().transform(numberEmptyValueTransform).min(1).max(99), + growth_rate: number() + .integer() + .transform(numberEmptyValueTransform) + .min(1) + .max(99) + .nullable(true), split_ratio: string().oneOf(splitRatioOptions).nullable(true), time_to_split: string().nullable(true), gender: string().required('This is a required field').oneOf(genderOptions), race: string().required('This is a required field').nullable(true).oneOf(raceOptions), - age_at_diagnosis: number().integer().transform(numberEmptyValueTransform).min(0).max(99), + age_at_diagnosis: number() + .integer() + .transform(numberEmptyValueTransform) + .min(0) + .max(99) + .nullable(true), age_at_sample_acquisition: number() .integer() .transform(numberEmptyValueTransform) diff --git a/cms/variant-migrations/migrate-mongo-config.js b/cms/variant-migrations/config.js similarity index 100% rename from cms/variant-migrations/migrate-mongo-config.js rename to cms/variant-migrations/config.js diff --git a/data_model/bin/fake.js b/data_model/bin/fake.js index fab091c3..c6e07997 100644 --- a/data_model/bin/fake.js +++ b/data_model/bin/fake.js @@ -56,7 +56,7 @@ let main = async () => { let variants = createVariants(models); let prepBulk = (docs, _index, _type) => - docs.map(doc => [ + docs.map((doc) => [ { index: { _index, diff --git a/data_model/package.json b/data_model/package.json index 16098d5a..8dc8b6fc 100644 --- a/data_model/package.json +++ b/data_model/package.json @@ -24,10 +24,9 @@ "yargs": "^15.4.1" }, "dependencies": { - "@elastic/elasticsearch": "~7.17.0", "babel-polyfill": "^6.26.0", "faker": "^4.1.0", - "lodash": "^4.17.21", + "lodash": "^4.18.1", "omit-deep": "^0.3.0", "ora": "^2.1.0", "randexp": "^0.4.9", diff --git a/docker-compose.yml b/docker-compose.yml index 56081978..231571da 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,22 +9,17 @@ services: - 27017:27017 volumes: - ./docker/mongo/data:/data/db - elasticsearch: - image: 'elasticsearch:7.17.6' + opensearch: + container_name: hcmi-opensearch + image: 'opensearchproject/opensearch:latest' ports: - 9200:9200 - 9300:9300 volumes: - - ./docker/elasticsearch/data:/usr/share/elasticsearch/data + - ./docker/opensearch/data:/usr/share/opensearch/data environment: - discovery.type=single-node - - cluster.name=workflow.elasticsearch + - cluster.name=workflow.opensearch - cluster.routing.allocation.disk.threshold_enabled=false - - 'ES_JAVA_OPTS=-Xms512m -Xmx2048m' - - search.max_buckets=65535 - kibana: - image: 'kibana:7.17.6' - depends_on: - - elasticsearch - ports: - - 5601:5601 + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx2048m + - DISABLE_SECURITY_PLUGIN=true diff --git a/docker/elasticsearch/config/elasticsearch.yml b/docker/elasticsearch/config/elasticsearch.yml deleted file mode 100644 index f0693eec..00000000 --- a/docker/elasticsearch/config/elasticsearch.yml +++ /dev/null @@ -1,10 +0,0 @@ -http.host: 0.0.0.0 - -# Uncomment the following lines for a production cluster deployment -#transport.host: 0.0.0.0 -#discovery.zen.minimum_master_nodes: 1 - -http.cors.enabled: true -http.cors.allow-origin: '*' -http.cors.allow-headers: Content-Type, Access-Control-Allow-Headers, Authorization, X-Requested-With -http.cors.allow-credentials: true diff --git a/docker/elasticsearch/config/log4j2.properties b/docker/elasticsearch/config/log4j2.properties deleted file mode 100644 index 46877d0d..00000000 --- a/docker/elasticsearch/config/log4j2.properties +++ /dev/null @@ -1,9 +0,0 @@ -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console diff --git a/elasticsearch/arranger_metadata/base.json b/elasticsearch/arranger_metadata/base.json index fb10e8f1..08954c28 100644 --- a/elasticsearch/arranger_metadata/base.json +++ b/elasticsearch/arranger_metadata/base.json @@ -1,4 +1,4 @@ { "documentType": "model", - "index": "hcmi" + "esIndex": "hcmi" } diff --git a/elasticsearch/arranger_metadata/extended.json b/elasticsearch/arranger_metadata/extended.json index ea81bc9e..30c9aa53 100644 --- a/elasticsearch/arranger_metadata/extended.json +++ b/elasticsearch/arranger_metadata/extended.json @@ -2,258 +2,277 @@ "extended": [ { "fieldName": "type", - "type": "keyword", + "type": "all", + "displayType": "keyword", "displayName": "Model Type", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "split_ratio", - "type": "keyword", + "type": "all", + "displayType": "keyword", "displayName": "Split Ratio", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { - "unit": null, + "unit": "", "displayValues": {}, "quickSearchEnabled": false, "fieldName": "time_to_split", "displayName": "Time to Split", - "active": false, + "isActive": false, "isArray": false, - "type": "keyword", + "type": "all", + "displayType": "keyword", "primaryKey": false, "rangeStep": 1 }, { "fieldName": "growth_rate", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "Doubling Time", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "molecular_characterizations", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Available Molecular Characterizations", - "active": false, + "isActive": false, "isArray": true, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { - "unit": null, + "unit": "", "displayValues": {}, "quickSearchEnabled": false, "fieldName": "tissue_type", "displayName": "Tissue Status", - "active": false, + "isActive": false, "isArray": false, - "type": "keyword", + "displayType": "keyword", + "type": "all", "primaryKey": false, "rangeStep": 1 }, - { "fieldName": "matched_models", + "displayType": "nested", "type": "nested", "displayName": "Multiple Models", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "matched_models.name", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Multiple Models From This Patient", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "matched_models.tissue_type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Multiple Model Tissue Status", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "matched_models.hits.total", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "Has Multiple Models", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "has_matched_models", + "displayType": "boolean", "type": "boolean", "displayName": "Has Multiple Models", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": { "true": "Yes", "false": "No" - } + }, + "rangeStep": 0 }, { "fieldName": "matched_models_list", - "type": "string", + "displayType": "string", + "type": "list", "displayName": "Has Multiple Models", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", + "displayValues": {}, "rangeStep": 1 }, - { "fieldName": "gender", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Sex", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, - { "fieldName": "race", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Race", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "age_at_diagnosis", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "Age At Diagnosis (Years)", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "age_at_sample_acquisition", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "Age At Acquisition (Years)", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, - { "fieldName": "disease_status", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Disease Status", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "vital_status", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Vital Status", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "neoadjuvant_therapy", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Neoadjuvant Therapy", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "therapy", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Therapy", - "active": false, + "isActive": false, "isArray": true, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "chemotherapeutic_drugs", + "displayType": "boolean", "type": "boolean", "displayName": "Chemotherapeutic Drug List Available", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": { "true": "Yes", "false": "No" @@ -262,623 +281,673 @@ }, { "fieldName": "clinical_diagnosis", - "type": "object", + "displayType": "object", + "type": "nested", "displayName": "Clinical Diagnosis", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "clinical_diagnosis.clinical_tumor_diagnosis", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Clinical Tumor Diagnosis", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "clinical_diagnosis.histological_type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Histological Subtype", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "primary_site", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Primary Site", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "clinical_diagnosis.site_of_sample_acquisition", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Acquisition Site", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "tnm_stage", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "TNM Stage", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "clinical_diagnosis.clinical_stage_grouping", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Clinical Stage Grouping", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "clinical_diagnosis.tumor_histological_grade", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Histological Grade", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "updatedAt", + "displayType": "date", "type": "date", "displayName": "Date Updated", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "date_of_availability", + "displayType": "date", "type": "date", "displayName": "Date Of Availability", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "licensing_required", + "displayType": "boolean", "type": "boolean", "displayName": "Licensing Required For Commercial Use", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": { "true": "Yes", "false": "No" - } + }, + "rangeStep": 0 }, { "fieldName": "createdAt", + "displayType": "date", "type": "date", "displayName": "Date Created", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files", + "displayType": "nested", "type": "nested", "displayName": "Files", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files.file_id", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "File ID", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files.file_name", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Files File Name", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files.file_type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "File Type", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files.magnification", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Files Magnification", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files.passage_number", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Files Passage Number", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "files.scale_bar_length", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Files Scale Bar Length", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "name", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Name", - "active": false, + "isActive": false, "isArray": false, "primaryKey": true, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { - "unit": null, + "unit": "", "displayValues": {}, "quickSearchEnabled": false, "fieldName": "distributor_part_number", "displayName": "Link To Distributor", - "active": false, + "isActive": false, "isArray": false, - "type": "keyword", + "displayType": "keyword", + "type": "all", "primaryKey": false, "rangeStep": 1 }, { "fieldName": "proteomics_url", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Link To Proteomics Data", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "source_model_url", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Link to Model Details", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "source_sequence_url", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Link To Sequencing Data", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "somatic_maf_url", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Link To Masked Somatic MAF", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "status", - "type": "text", + "displayType": "keyword", + "type": "all", "displayName": "Status", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, - { "fieldName": "variants", + "displayType": "nested", "type": "nested", "displayName": "Variants", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.assessment_type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Assessment Type", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.category", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Category", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.expression_level", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Expression Level", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.genes", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Genes", - "active": false, + "isActive": false, "isArray": true, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.name", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Variant", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Variants Type", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, - { "fieldName": "genomic_variants", + "displayType": "nested", "type": "nested", "displayName": "Genomic Variants", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "variants.type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Variants Type", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, - { "fieldName": "autocomplete", - "type": "text", + "displayType": "keyword", + "type": "all", "displayName": "Autocomplete", - "active": false, + "isActive": false, "isArray": true, "primaryKey": false, "quickSearchEnabled": true, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "expanded", + "displayType": "boolean", "type": "boolean", "displayName": "Expansion Status", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": { "true": "Expanded", "false": "Unexpanded" - } + }, + "rangeStep": 0 }, { "fieldName": "genomic_variants", + "displayType": "nested", "type": "nested", "displayName": "Genomic Variants", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "genomic_variants.gene", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Gene", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "genomic_variants.transcript_id", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Research Somatic Variant Transcript ID", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "genomic_variants.variant_id", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Research Somatic Variant ID", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "genomic_variants.synonyms", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Gene Synonyms", - "active": false, + "isActive": false, "isArray": true, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "genomic_variants.classification", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Research Somatic Variant Type", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "genomic_variants.consequence_type", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Consequence", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata", - "type": "object", + "displayType": "object", + "type": "nested", "displayName": "Gene Metadata", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata.genes", - "type": "keyword", + "displayType": "keyword", + "type": "all", "displayName": "Mutated Genes", - "active": false, + "isActive": false, "isArray": true, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata.genes_count", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "# Mutated Genes", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata.mutated_genes_count", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "# Mutated Genes", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata.genomic_variant_count", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "# Research Somatic Variants", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata.clinical_variant_count", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "# Clinical Variants", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 }, { "fieldName": "gene_metadata.histopathological_variant_count", - "type": "long", + "displayType": "long", + "type": "number", "displayName": "# Histo-pathological Biomarkers", - "active": false, + "isActive": false, "isArray": false, "primaryKey": false, "quickSearchEnabled": false, - "unit": null, + "unit": "", "displayValues": {}, "rangeStep": 1 } diff --git a/elasticsearch/arranger_metadata/facets.json b/elasticsearch/arranger_metadata/facets.json index 35c34b33..f4a59b42 100644 --- a/elasticsearch/arranger_metadata/facets.json +++ b/elasticsearch/arranger_metadata/facets.json @@ -3,278 +3,381 @@ "aggregations": [ { "fieldName": "primary_site", + "displayName": "Primary Site", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "genomic_variants__classification", + "displayName": "Genomic Variants Classification", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "genomic_variants__consequence_type", + "displayName": "Genomic Variants Consequence Type", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "type", + "displayName": "Type", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "has_matched_models", + "displayName": "Has Matched Models", + "displayType": "boolean", "show": true, - "active": true + "isActive": true }, { "fieldName": "expanded", + "displayName": "Expanded", + "displayType": "boolean", "show": false, - "active": true + "isActive": true }, { "fieldName": "clinical_diagnosis__site_of_sample_acquisition", + "displayName": "Site of Sample Acquisition", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "clinical_diagnosis__clinical_tumor_diagnosis", + "displayName": "Clinical Tumor Diagnosis", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "clinical_diagnosis__clinical_stage_grouping", + "displayName": "Clinical Stage Grouping", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "tissue_type", + "displayName": "Tissue Type", + "displayType": "keyword", "show": true, - "active": true - }, - { - "fieldName": "clinical_diagnosis__histological_type", - "show": true, - "active": true + "isActive": true }, { "fieldName": "clinical_diagnosis__tumor_histological_grade", + "displayName": "Tumor Histological Grade", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "age_at_sample_acquisition", + "displayName": "Age at Sample Acquisition", + "displayType": "long", "show": false, - "active": true + "isActive": true }, { "fieldName": "age_at_diagnosis", + "displayName": "Age at Diagnosis", + "displayType": "long", "show": true, - "active": true + "isActive": true }, { "fieldName": "disease_status", + "displayName": "Disease Status", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "gender", + "displayName": "Gender", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "molecular_characterizations", + "displayName": "Molecular Characterizations", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "neoadjuvant_therapy", + "displayName": "Neoadjuvant Therapy", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "chemotherapeutic_drugs", + "displayName": "Chemotherapeutic Drugs", + "displayType": "boolean", "show": true, - "active": true + "isActive": true }, { "fieldName": "createdAt", + "displayName": "Created At", + "displayType": "date", "show": false, - "active": true + "isActive": true }, { "fieldName": "date_of_availability", + "displayName": "Date of Availability", + "displayType": "date", "show": false, - "active": true + "isActive": true }, { "fieldName": "updatedAt", + "displayName": "Updated At", + "displayType": "date", "show": false, - "active": true + "isActive": true }, { "fieldName": "growth_rate", + "displayName": "Growth Rate", + "displayType": "long", "show": false, - "active": true + "isActive": true }, { "fieldName": "licensing_required", + "displayName": "Licensing Required", + "displayType": "boolean", "show": true, - "active": true + "isActive": true }, { "fieldName": "name", + "displayName": "Name", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "race", + "displayName": "Race", + "displayType": "keyword", "show": true, - "active": true + "isActive": true }, { "fieldName": "variants__genes", + "displayName": "Variants Genes", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "source_model_url", + "displayName": "Source Model URL", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "files__file_id", + "displayName": "File ID", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "files__file_name", + "displayName": "File Name", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "files__file_type", + "displayName": "File Type", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "matched_models__name", + "displayName": "Matched Models Name", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "matched_models__tissue_type", + "displayName": "Matched Models Tissue Type", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "proteomics_url", + "displayName": "Proteomics URL", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "source_sequence_url", + "displayName": "Source Sequence URL", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "somatic_maf_url", + "displayName": "Somatic MAF URL", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "split_ratio", + "displayName": "Split Ratio", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "therapy", + "displayName": "Therapy", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "tnm_stage", + "displayName": "TNM Stage", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "variants__assessment_type", + "displayName": "Variant Assessment Type", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "variants__category", + "displayName": "Variant Category", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "variants__expression_level", + "displayName": "Variant Expression Level", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "variants__name", + "displayName": "Variant Name", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "variants__type", + "displayName": "Variant Type", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "vital_status", + "displayName": "Vital Status", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "autocomplete", + "displayName": "Autocomplete", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "files__magnification", + "displayName": "File Magnification", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "files__passage_number", + "displayName": "File Passage Number", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "files__scale_bar_length", + "displayName": "File Scale Bar Length", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "status", + "displayName": "Status", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "distributor_part_number", + "displayName": "Distributor Part Number", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "time_to_split", + "displayName": "Time to Split", + "displayType": "keyword", "show": false, - "active": true + "isActive": true }, { "fieldName": "matched_models_list", + "displayName": "Matched Models List", + "displayType": "string", "show": false, - "active": true + "isActive": true }, { "fieldName": "gene_metadata__genes", + "displayName": "Gene Metadata", + "displayType": "object", "show": false, - "active": true + "isActive": true }, { "fieldName": "gene_metadata__mutated_genes", + "displayName": "Mutated Genes", + "displayType": "keyword", "show": false, - "active": true + "isActive": true } ] } diff --git a/elasticsearch/arranger_metadata/matchbox.json b/elasticsearch/arranger_metadata/matchbox.json index 933d1270..7e686162 100644 --- a/elasticsearch/arranger_metadata/matchbox.json +++ b/elasticsearch/arranger_metadata/matchbox.json @@ -2,35 +2,35 @@ "matchbox": [ { "keyField": "createdAt", - "field": "", + "fieldName": "", "displayName": "models", "searchFields": [], "isActive": false }, { "keyField": null, - "field": "variants", + "fieldName": "variants", "displayName": "Variants", "searchFields": [], "isActive": false }, { "keyField": null, - "field": "files", + "fieldName": "files", "displayName": "Files", "searchFields": [], "isActive": false }, { "keyField": null, - "field": "multiple_models", + "fieldName": "multiple_models", "displayName": "Multiple Models", "searchFields": [], "isActive": false }, { "keyField": null, - "field": "gene_metadata", + "fieldName": "gene_metadata", "displayName": "Gene Metadata", "searchFields": [], "isActive": false diff --git a/elasticsearch/arranger_metadata/table.json b/elasticsearch/arranger_metadata/table.json index 8d1397bf..1e712f13 100644 --- a/elasticsearch/arranger_metadata/table.json +++ b/elasticsearch/arranger_metadata/table.json @@ -5,390 +5,581 @@ "defaultSorting": [ { "fieldName": "gene_metadata.genomic_variant_count", - "desc": true + "desc": true, + "isActive": false } ], "columns": [ { "fieldName": "name", + "displayName": "Name", "accessor": "name", "show": true, - "type": "entity", + "displayType": "entity", "sortable": true, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "primary_site", + "displayName": "Primary Site", "accessor": "primary_site", "show": true, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "clinical_diagnosis.clinical_tumor_diagnosis", "accessor": "clinical_diagnosis.clinical_tumor_diagnosis", "show": true, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Clinical Tumor Diagnosis", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "clinical_diagnosis.histological_type", "accessor": "clinical_diagnosis.histological_type", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Histological Type", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "tissue_type", "accessor": "tissue_type", "show": true, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Tissue Type", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "clinical_diagnosis.site_of_sample_acquisition", "accessor": "clinical_diagnosis.site_of_sample_acquisition", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Site of Sample Acquisition", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "gender", "accessor": "gender", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Gender", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "race", "accessor": "race", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Race", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "age_at_sample_acquisition", "accessor": "age_at_sample_acquisition", "show": true, - "type": "age_at_sample_acquisition", + "displayType": "age_at_sample_acquisition", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Age at Sample Acquisition", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "age_at_diagnosis", "accessor": "age_at_diagnosis", "show": true, - "type": "number", + "displayType": "number", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Age at Diagnosis", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "disease_status", "accessor": "disease_status", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Disease Status", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "vital_status", "accessor": "vital_status", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Vital Status", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "tnm_stage", "accessor": "tnm_stage", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "TNM Stage", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "clinical_diagnosis.clinical_stage_grouping", "accessor": "clinical_diagnosis.clinical_stage_grouping", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Clinical Stage Grouping", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "clinical_diagnosis.tumor_histological_grade", "accessor": "clinical_diagnosis.tumor_histological_grade", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Tumor Histological Grade", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "matched_models_list", "accessor": "matched_models_list", "show": true, - "type": "matched_models", + "displayType": "matched_models", "sortable": false, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Matched Models List", + "displayFormat": "", + "displayValues": {}, + "isArray": true, + "isActive": false }, { "fieldName": "neoadjuvant_therapy", "accessor": "neoadjuvant_therapy", "show": false, - "type": "keyword", + "displayType": "keyword", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Neoadjuvant Therapy", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "chemotherapeutic_drugs", "accessor": "chemotherapeutic_drugs", "show": false, - "type": "boolean", + "displayType": "boolean", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Chemotherapeutic Drugs", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "therapy", "accessor": "therapy", "show": false, - "type": "list", + "displayType": "list", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Therapy", + "displayFormat": "", + "displayValues": {}, + "isArray": true, + "isActive": false }, { "fieldName": "molecular_characterizations", "accessor": "molecular_characterizations", "show": false, - "type": "list", + "displayType": "list", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Molecular Characterizations", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "distributor_part_number", "accessor": "distributor_part_number", "show": false, - "type": "distributor_link", + "displayType": "distributor_link", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Distributor Part Number", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "type", "accessor": "type", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Type", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "licensing_required", "accessor": "licensing_required", "show": false, - "type": "boolean", + "displayType": "boolean", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Licensing Required", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "expanded", "accessor": "expanded", "show": true, - "type": "expanded", + "displayType": "expanded", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Expanded", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "gene_metadata.genes_count", "accessor": "gene_metadata.genes_count", "show": false, - "type": "genes_count", + "displayType": "genes_count", "sortable": true, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Gene Count", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "gene_metadata.mutated_genes_count", "accessor": "gene_metadata.mutated_genes_count", "show": true, - "type": "genes_count", + "displayType": "genes_count", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Mutated Genes Count", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "gene_metadata.genomic_variant_count", "accessor": "gene_metadata.genomic_variant_count", "show": true, - "type": "number", + "displayType": "number", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Genomic Variant Count", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "gene_metadata.clinical_variant_count", "accessor": "gene_metadata.clinical_variant_count", "show": true, - "type": "number", + "displayType": "number", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Clinical Variant Count", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "gene_metadata.histopathological_variant_count", "accessor": "gene_metadata.histopathological_variant_count", "show": true, - "type": "histo_variant_count", + "displayType": "histo_variant_count", "sortable": true, "canChangeShow": true, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Histopathological Variant Count", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "split_ratio", "accessor": "split_ratio", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Split Ratio", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "growth_rate", "accessor": "growth_rate", "show": false, - "type": "number", + "displayType": "number", "sortable": true, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Growth Rate", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "time_to_split", "accessor": "time_to_split", "show": false, - "type": "string", + "displayType": "string", "sortable": true, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Time to Split", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "createdAt", "accessor": "createdAt", "show": false, - "type": "date", + "displayType": "date", "sortable": false, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Created At", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "date_of_availability", "accessor": "date_of_availability", "show": false, - "type": "date", + "displayType": "date", "sortable": false, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Date of Availability", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "source_model_url", "accessor": "source_model_url", "show": false, - "type": "string", + "displayType": "string", "sortable": false, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Source Model URL", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "source_sequence_url", "accessor": "source_sequence_url", "show": false, - "type": "string", + "displayType": "string", "sortable": false, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Source Sequence URL", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "somatic_maf_url", "accessor": "somatic_maf_url", "show": false, - "type": "string", + "displayType": "string", "sortable": false, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Somatic MAF URL", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false }, { "fieldName": "proteomics_url", "accessor": "proteomics_url", "show": false, - "type": "string", + "displayType": "string", "sortable": false, "canChangeShow": false, - "query": null, - "jsonPath": null + "query": "", + "jsonPath": "", + "displayName": "Proteomics URL", + "displayFormat": "", + "displayValues": {}, + "isArray": false, + "isActive": false } ], "timestamp": "2019-10-11T20:36:07.801Z" diff --git a/elasticsearch/lastUpdated.json b/elasticsearch/lastUpdated.json new file mode 100644 index 00000000..b4238056 --- /dev/null +++ b/elasticsearch/lastUpdated.json @@ -0,0 +1,10 @@ +{ + "mappings": { + "properties": { + "date": { + "type": "date", + "format": "epoch_millis" + } + } + } +} diff --git a/jenkinsfile.ecs01 b/jenkinsfile.ecs01 new file mode 100644 index 00000000..d60c35fb --- /dev/null +++ b/jenkinsfile.ecs01 @@ -0,0 +1,138 @@ +#!groovy +def CMS_PACKAGE_TYPE='cms' +def API_PACKAGE_TYPE='api' +def UI_PACKAGE_TYPE='ui' + +void failSafeBuild(configId, packageType){ + try { + env.BUILD_STEP_SUCCESS = 'no' + def targetLocation = '' + + // set target location based on package type + if (packageType == 'cms'){ + targetLocation = './cms/pm2.config.js' + + configFileProvider([ + configFile(fileId: configId, targetLocation: targetLocation), + configFile(fileId: configId + '-migration', targetLocation: './cms/variant-migrations/config.js'), + ]) { + sh ''' + umask 022 + portal-ci/build_stage/build.sh portal-ci ''' + packageType + ''' + ''' + env.BUILD_STEP_SUCCESS = 'yes' + } + return + } + + if (packageType == 'api') { + targetLocation = './api/pm2.config.js' + } else if (packageType == 'ui') { + targetLocation = './ui/.env' + } + + configFileProvider([ + configFile(fileId: configId, targetLocation: targetLocation) + ]) { + sh ''' + umask 022 + portal-ci/build_stage/build.sh portal-ci ''' + packageType + ''' + ''' + env.BUILD_STEP_SUCCESS = 'yes' + } + + } catch (err) { + env.BUILD_STEP_SUCCESS = 'no' + echo "Required configuration for $packageType not found. Skipping the build for $packageType." + } +} + +void getPipelineResult (){ + script { + // fail the build if all deployment stages were skipped + if(env.DEV_DEPLOYMENT_STATUS == null && env.STAGING_DEPLOYMENT_STATUS == null && env.PRD_DEPLOYMENT_STATUS == null) { + echo 'Build failed because application was not deployed to any environment.' + echo 'Please make sure Jenkins has all required configuration files and variables.' + currentBuild.result = 'FAILURE' + } else { + echo 'Build is considered successful because application was successfully deployed in at least one target environment.' + } + } +} + +node ('ecs-agent') { + configFileProvider([configFile(fileId: 'hcmi-env-config', variable: 'FILE')]) { + echo "FILE=$FILE" + load "$FILE" + } +} +pipeline { + agent { label 'ecs-agent' } + stages{ + stage('Get Code') { + steps { + echo "Workspace directory is ${env.WORKSPACE}" + deleteDir() + checkout ([ + $class: 'GitSCM', + branches: scm.branches, + doGenerateSubmoduleConfigurations: scm.doGenerateSubmoduleConfigurations, + extensions: [[$class: 'CloneOption', noTags: false, shallow: false, depth: 0, reference: '']], + userRemoteConfigs: scm.userRemoteConfigs, + ]) + script { + tag=sh(returnStdout: true, script: "git tag -l --points-at HEAD").trim() + env.tag = tag + } + } + } + stage('GetOpsScripts') { + steps { + echo "GETTING SCRIPTS" + sh ''' + umask 022 + git clone '''+PORTAL_CI_URL+''' + ''' + } + } + stage('Build Dev') { + steps { + failSafeBuild('hcmi-cms-dev-config', CMS_PACKAGE_TYPE) + failSafeBuild('hcmi-api-dev-config', API_PACKAGE_TYPE) + failSafeBuild('hcmi-ui-dev-config', UI_PACKAGE_TYPE) + } + } + stage('Deploy Dev') { + when{ + environment name: 'BUILD_STEP_SUCCESS', value: 'yes' + } + steps { + echo "DEPLOYING TO DEVELOPMENT: (${env.BUILD_URL})" + sshagent (credentials: ["$DEV_CREDS"]) { + sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$CMS_DEV_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${CMS_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$CMS_DEV_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$CMS_DEV_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh dev $BUILD_NUMBER $CMS_PACKAGE_TYPE\"" + ) + sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$API_DEV_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${API_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$API_DEV_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$API_DEV_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh dev $BUILD_NUMBER $API_PACKAGE_TYPE\"" + ) + sh (returnStdout: false, script: "ssh -o StrictHostKeyChecking=no $APP_USER@$UI_DEV_SERVER \"set -x; if [ ! -d $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ ]; then mkdir -p $REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ || exit \$?; fi\" && scp ${UI_PACKAGE_TYPE}.tar portal-ci/deploy_stage/deploy.sh $APP_USER@$UI_DEV_SERVER:$REMOTE_DIR/hcmi/deploy/$BUILD_NUMBER/ && ssh -o StrictHostKeyChecking=no $APP_USER@$UI_DEV_SERVER \"set -x; cd $REMOTE_DIR/hcmi && bash deploy/$BUILD_NUMBER/deploy.sh dev $BUILD_NUMBER $UI_PACKAGE_TYPE\"" + ) + } + echo "DEPLOYED TO DEVELOPMENT: (${env.BUILD_URL})" + script { + env.DEV_DEPLOYMENT_STATUS = 'SUCCESS' + } + + } + post { + failure { + echo "Deploy Failed: Branch '${env.BRANCH_NAME} [${env.BUILD_NUMBER}]' (${env.BUILD_URL})" + } + } + } + } + + post{ + always { + getPipelineResult() + } + } +} diff --git a/package.json b/package.json index 27cd9b77..92eb16ca 100644 --- a/package.json +++ b/package.json @@ -40,8 +40,7 @@ "@babel/eslint-plugin": "^7.27.1", "@babel/plugin-proposal-optional-chaining": "^7.21.0", "@babel/preset-react": "^7.27.1", - "@elastic/elasticsearch": "~7.17.0", - "axios": "^1.7.9", + "axios": "^1.16.0", "eslint": "^9.39.1", "eslint-config-prettier": "^10.1.8", "eslint-config-react-app": "7.0.1", @@ -64,8 +63,8 @@ }, "resolutions": { "js-yaml": "^4.1.1", - "mongoose": "^6.13.8", "node-fetch": "^2.6.7", + "lodash": "^4.18.1", "xml2js": "^0.5.0", "node-forge": "^1.3.2" } diff --git a/scripts/initializeEs.js b/scripts/initializeEs.js index 9aa2511a..b4601e5f 100644 --- a/scripts/initializeEs.js +++ b/scripts/initializeEs.js @@ -1,12 +1,20 @@ import esUtils from './utils/esUtils.js'; const run = async () => { - /** initialize search index */ - await esUtils.createModelsIndex(); - await esUtils.configureArrangerSets(); + const { + createLastUpdatedIndex, + createModelsIndex, + configureArrangerSets, + createGenesIndex, + createVariantsIndex, + } = esUtils; - await esUtils.createGenesIndex(); - await esUtils.createVariantsIndex(); + /** initialize search index */ + await createLastUpdatedIndex(); + await createModelsIndex(); + await configureArrangerSets(); + await createGenesIndex(); + await createVariantsIndex(); }; run(); diff --git a/scripts/initializeExpanded.js b/scripts/initializeExpanded.js index e3e4b5ea..7d1e970f 100644 --- a/scripts/initializeExpanded.js +++ b/scripts/initializeExpanded.js @@ -10,7 +10,7 @@ const conn = mongoose.createConnection(process.env.MONGODB_URI || 'mongodb://loc conn.once('open', async () => { try { const models = await conn.db - .collection(process.env.MONGO_COLLECTION) + .collection(MONGO_COLLECTION) .find({ expanded: { $exists: false } }); console.log('Models to update:'); let model; @@ -19,7 +19,7 @@ conn.once('open', async () => { } console.log('\nUpdating now...'); const updateResult = await conn.db - .collection(process.env.MONGO_COLLECTION) + .collection(MONGO_COLLECTION) .updateMany({ expanded: { $exists: false } }, { $set: { expanded: true } }); console.log('Models updated:', updateResult.modifiedCount); diff --git a/scripts/utils/esUtils.js b/scripts/utils/esUtils.js index 83e0d477..a2d0de6a 100644 --- a/scripts/utils/esUtils.js +++ b/scripts/utils/esUtils.js @@ -1,4 +1,10 @@ -import es from '@elastic/elasticsearch'; +import getSearchClient from '../../cms/src/services/search-client/client.js'; + +/** Search index settings and mappings **/ +import modelsIndexConfig from '../../elasticsearch/modelsIndex.json' with { type: "json" }; +import updateIndexConfig from '../../elasticsearch/lastUpdated.json' with { type: "json" }; +import genesIndexConfig from '../../elasticsearch/genesIndex.json' with { type: "json" }; +import variantsIndexConfig from '../../elasticsearch/variantsIndex.json' with { type: "json" }; const pm2Path = process.env.CMS_CONFIG || '../../cms/pm2.config.js'; const pm2Env = process.env.ENV; @@ -10,27 +16,28 @@ const pm2ConfigGeneric = (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0].env) || {}; const pm2ConfigForEnv = (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0][`env_${pm2Env}`]) || {}; - const pm2 = { ...pm2ConfigGeneric, ...pm2ConfigForEnv }; -/** Search index settings and mappings **/ -import modelsIndexConfig from '../../elasticsearch/modelsIndex.json' with { type: "json" }; -import genesIndexConfig from '../../elasticsearch/genesIndex.json' with { type: "json" }; -import variantsIndexConfig from '../../elasticsearch/variantsIndex.json' with { type: "json" }; -const modelsIndexName = process.env.ES_INDEX || pm2.ES_INDEX || 'hcmi'; const esHost = process.env.ES_HOST || `${pm2.ES_HOST}:${pm2.ES_PORT}`; +const updateIndexName = process.env.ES_UPDATE_INDEX || pm2.ES_UPDATE_INDEX || 'hcmi-update'; +const modelsIndexName = process.env.ES_INDEX || pm2.ES_INDEX || 'hcmi'; +const user = pm2?.ES_USER || process.env.ES_USER || ''; +const password = pm2?.ES_PASS || process.env.ES_PASS || ''; +const clientType = pm2?.SEARCH_CLIENT_TYPE || process.env.SEARCH_CLIENT_TYPE || 'opensearch'; const GENES_INDEX = 'genes'; const VARIANTS_INDEX = 'genomic_variants'; -const client = new es.Client({ - node: esHost, -}); - /* ******** Index creation and deletion ******** */ const createIndex = async (index, config) => { try { console.log(`\nCreating index: ${index}`); + const client = await getSearchClient({ + node: esHost, + user, + password, + clientType + }); await client.indices.create({ index, body: config, @@ -46,11 +53,20 @@ const createIndex = async (index, config) => { const deleteIndex = async index => { try { console.log(`\nDeleting existing index (if present): ${index}`); + const client = await getSearchClient({ + node: esHost, + user, + password, + clientType + }); await client.indices.delete({ index }); } catch (e) {} }; /* ******* Models Index ******** */ +const createLastUpdatedIndex = async () => + await createIndex(updateIndexName, updateIndexConfig); + const createModelsIndex = async () => await createIndex(modelsIndexName, modelsIndexConfig); @@ -71,6 +87,12 @@ const deleteVariantsIndex = async () => await deleteIndex(VARIANTS_INDEX); const updateIndex = async ({ index, settings = {}, mappings = {} } = {}) => { try { console.log('Updating mapping for:', index); + const client = await getSearchClient({ + node: esHost, + user, + password, + clientType + }); await client.indices.close({ index, }); @@ -109,10 +131,22 @@ const updateSearchIndices = async () => { const configureArrangerSets = async () => { try { console.log(`\nDeleting existing index (if present): arranger-sets`); + const client = await getSearchClient({ + node: esHost, + user, + password, + clientType + }); await client.indices.delete({ index: `arranger-sets` }); } catch (e) {} try { console.log(`Creating index: arranger-sets`); + const client = await getSearchClient({ + node: esHost, + user, + password, + clientType + }); await client.indices.create({ index: 'arranger-sets', body: { @@ -152,6 +186,7 @@ const configureArrangerSets = async () => { const esUtils = { config: pm2, + createLastUpdatedIndex, createModelsIndex, deleteModelsIndex, createGenesIndex, diff --git a/scripts/utils/republishUtils.js b/scripts/utils/republishUtils.js index 53303479..2acb1c4b 100644 --- a/scripts/utils/republishUtils.js +++ b/scripts/utils/republishUtils.js @@ -5,25 +5,30 @@ process.env = esUtils.config; import 'babel-polyfill'; import mongoose from 'mongoose'; -import { publishModel } from '../../cms/src/services/elastic-search/publish.js'; -import { ModelES } from '../../cms/src/services/elastic-search/common/schemas/model.js'; -import '../../cms/src/schemas/variant.js'; -import '../../cms/src/schemas/matchedModels.js'; import { modelStatus } from '../../cms/src/helpers/modelStatus.js'; +import { publishModel } from '../../cms/src/services/search-client/publish.js'; +import indexLastUpdated from '../../cms/src/services/search-client/indexLastUpdated.js'; -import indexEsUpdate from '../../cms/src/services/elastic-search/update.js'; +import '../../cms/src/schemas/variant.js'; +import '../../cms/src/schemas/matchedModels.js'; export const republishModels = async () => { console.log('Connecting to MongoDB...'); // Connect to database - await mongoose.connect(esUtils.config.MONGODB_URI); + const { MONGODB_URI, MONGO_COLLECTION = 'models' } = esUtils.config; + + const client = await mongoose.connect(MONGODB_URI); console.log('\nConnected!'); - const models = await ModelES.find({ - status: { $in: [modelStatus.published, modelStatus.unpublishedChanges] }, - }); + const modelCollection = await client.connection.db.collection(MONGO_COLLECTION); + + const models = await modelCollection + .find({ + status: { $in: [modelStatus.published, modelStatus.unpublishedChanges] }, + }) + .toArray(); console.log('\nSearching for models to publish...'); const names = models.map((i) => i.name); @@ -33,7 +38,7 @@ export const republishModels = async () => { await publishModel({ name: model.name }); } - indexEsUpdate(); + indexLastUpdated(); mongoose.disconnect(); }; diff --git a/ui/package.json b/ui/package.json index 1498346d..7541e3d8 100644 --- a/ui/package.json +++ b/ui/package.json @@ -14,7 +14,7 @@ "@emotion/styled": "^11.14.1", "@nivo/bar": "^0.88.0", "@nivo/pie": "^0.88.0", - "@overture-stack/arranger-components": "^3.0.2", + "@overture-stack/arranger-components": "^3.0.7", "@react-oauth/google": "^0.12.2", "@vitejs/plugin-react": "^5.0.1", "isbot": "^5.1.27", diff --git a/ui/src/components/Model.jsx b/ui/src/components/Model.jsx index cca29dca..b8e6e91c 100644 --- a/ui/src/components/Model.jsx +++ b/ui/src/components/Model.jsx @@ -289,7 +289,7 @@ const Model = ({ modelName }) => ( {({ state: queryState, modelImages = modelImageProcessor( - queryState.model?.files?.hits ? queryState.model.files.hits.edges : [], + queryState.model?.files?.hits ? queryState.model?.files?.hits?.edges : [], ), }) => { return ( diff --git a/ui/src/components/TableMatchedModelsCell.jsx b/ui/src/components/TableMatchedModelsCell.jsx index d80c06c9..210d1df4 100644 --- a/ui/src/components/TableMatchedModelsCell.jsx +++ b/ui/src/components/TableMatchedModelsCell.jsx @@ -4,13 +4,16 @@ import { useTableContext } from '@overture-stack/arranger-components'; const { stringify } = querystring; -const TableMatchedModelsCell = ({ row, savedSetsContext, value, history }) => { +const TableMatchedModelsCell = ({ row, savedSetsContext, history }) => { const { sorting } = useTableContext({ callerName: 'TableMatchedModelsCell', }); - const matches = (value && value.split(',')) || []; + const matches = row.original.matched_models_list?.length + ? row.original.matched_models_list.split(',') + : []; const matchCount = matches.length; - return matchCount > 1 ? ( + + return !!matchCount ? (