#1142 OpenSearch & AWS migration

api/package.json

@@ -21,7 +21,7 @@
     "helmet": "^3.15.0",
     "jszip": "^3.7.0",
     "map-stream": "^0.0.7",
-    "mongoose": "7.8.7",
+    "mongoose": "^8.23.0",
     "pino": "^9.13.1",
     "pino-http": "^10.5.0",
     "sharp": "0.34.4"

api/src/health.js

@@ -1,14 +1,9 @@
 import express from 'express';
-import elasticsearch from '@elastic/elasticsearch';
-
 import _ from 'lodash';
 
-const startTime = Date.now();
-
-const client = new elasticsearch.Client({
-  node: process.env.ES_URL,
-});
+import getClient from './services/searchClient.ts';
 
+const startTime = Date.now();
 const healthRouter = express.Router();
 
 healthRouter.get('/', async (req, res) => {
@@ -18,6 +13,7 @@ healthRouter.get('/', async (req, res) => {
 
 healthRouter.get('/es', async (req, res) => {
   try {
+    const client = await getClient();
     const status = await client.ping();
     if (_.get(status, 'statusCode') === 200) {
       const response = _.omit(status, 'meta');

api/src/lastUpdated.js

@@ -1,13 +1,17 @@
 // @ts-check
 
 import express from 'express';
-import esClient from './services/elasticsearch';
+
+import getClient from './services/searchClient.ts';
+import getLogger from './logger';
 
 const lastUpdatedRouter = express.Router();
+const logger = getLogger('lastUpdated Router');
 
 lastUpdatedRouter.get('/', async (req, res) => {
+  const searchClient = await getClient();
   try {
-    const response = await esClient.search({
+    const response = await searchClient.search({
       index: process.env.ES_UPDATE_INDEX,
       body: {
         query: {
@@ -23,8 +27,9 @@ lastUpdatedRouter.get('/', async (req, res) => {
         ],
       },
     });
-    return res.json(response.body.hits.hits[0]._source);
+    return res.json(response.body?.hits?.hits[0]?._source);
   } catch (error) {
+    logger.error(`Error retrieving last updated date from Arranger: ${error}`);
     return res.status(500).json({
       error: error,
     });

api/src/search.js

@@ -1,5 +1,5 @@
 import express from 'express';
-import esClient from './services/elasticsearch';
+import getClient from './services/searchClient.ts';
 import { get } from 'lodash';
 
 const GENES_INDEX = 'genes';
@@ -28,7 +28,8 @@ geneSearchRouter.get('/gene', async (req, res) => {
       },
     };
 
-    const response = await esClient.search({
+    const searchClient = await getClient();
+    const response = await searchClient.search({
       index: GENES_INDEX,
       body: { query },
     });

api/src/services/searchClient.ts

@@ -0,0 +1,8 @@
+import { getSearchClient } from '@overture-stack/arranger-server';
+
+const getClient = async () =>
+  await getSearchClient({
+    node: process.env.ES_URL || 'http://localhost:9200',
+  });
+
+export default getClient;

cms/package.json

@@ -32,6 +32,7 @@
     "lodash": "^4.17.21",
     "map-stream": "^0.0.7",
     "method-override": "^2.3.10",
+    "mongoosastic": "^5.0.0",
     "mongoose": "7.8.7",
     "mongoose-elasticsearch-xp": "^5.8.0",
     "morgan": "^1.10.1",

cms/src/helpers/dictionary.js

@@ -35,21 +35,21 @@ export const getDictionaryOptions = async () => {
 
   const output = dictionary.reduce((acc, field) => {
     let { name, values } = field;
-    acc[`${name}Options`] = values.map(i => i.value);
+    acc[`${name}Options`] = values.map((i) => i.value);
     return acc;
   }, {});
-  const ctd = dictionary.find(i => i.name === 'clinicalTumorDiagnosis');
+  const ctd = dictionary.find((i) => i.name === 'clinicalTumorDiagnosis');
 
   const ctdDependentOptions = {};
 
-  ctd.dependentValues.forEach(val => (ctdDependentOptions[val] = {}));
+  ctd.dependentValues.forEach((val) => (ctdDependentOptions[val] = {}));
 
-  ctd.values.forEach(val => {
+  ctd.values.forEach((val) => {
     const valueName = val.value;
 
-    val.dependents.forEach(dependent => {
+    val.dependents.forEach((dependent) => {
       const dependentName = dependent.name;
-      const dependentValues = dependent.values.map(val => val.value);
+      const dependentValues = dependent.values.map((val) => val.value);
 
       ctdDependentOptions[dependentName][valueName.toLowerCase()] = dependentValues;
     });
@@ -63,7 +63,7 @@ export const resetDraft = async () => {
   const dictionary = await Dictionary.findOne({}, {}, { sort: { created_at: -1 } });
   const draft = new Draft({ fields: dictionary.fields });
   await draft.save();
-  logger.audit({}, 'reset draft', 'Dictionary draft reset');
+  logger.info({}, 'reset draft', 'Dictionary draft reset');
   return await Draft.findOne({}, {}, { sort: { created_at: -1 } });
 };
 
@@ -77,12 +77,12 @@ export const publishDraft = async () => {
 
   // find edited values
   const edits = [];
-  draft.fields.forEach(field => {
+  draft.fields.forEach((field) => {
     if (field.dependentValues.length > 0) {
       // dependent field case
-      field.values.forEach(value => {
-        value.dependents.forEach(dependent => {
-          dependent.values.forEach(dependentValue => {
+      field.values.forEach((value) => {
+        value.dependents.forEach((dependent) => {
+          dependent.values.forEach((dependentValue) => {
             if (dependentValue.status === draftStatus.edited) {
               edits.push({
                 field: field.name,
@@ -97,7 +97,7 @@ export const publishDraft = async () => {
       });
     }
     // basic field case
-    field.values.forEach(value => {
+    field.values.forEach((value) => {
       if (value.status === draftStatus.edited) {
         edits.push({
           field: field.name,
@@ -119,19 +119,22 @@ export const publishDraft = async () => {
 
   const dictionary = new Dictionary({ fields: draft.fields });
   await dictionary.save();
-  logger.audit({ edits }, 'dictionary created', 'Created new dictionary from draft values');
+  logger.info({ edits }, 'dictionary created', 'Created new dictionary from draft values');
 
-  models.forEach(model => {
+  models.forEach((model) => {
     // Find these values in the models.
     let edited = false;
-    edits.forEach(edit => {
+    edits.forEach((edit) => {
       const editField = edit.dependentName ? edit.dependentName : edit.field;
 
       // - update those models and change their status
       if (editField === 'therapy' && model.therapy.includes(edit.original)) {
         // special case, therapy is an array
 
-        model.therapy.splice(model.therapy.indexOf(val => val === edit.original), 1);
+        model.therapy.splice(
+          model.therapy.indexOf((val) => val === edit.original),
+          1,
+        );
         model.therapy.push(edit.value);
         model.status =
           model.status === modelStatus.published ? modelStatus.unpublishedChanges : model.status;
@@ -148,7 +151,7 @@ export const publishDraft = async () => {
     });
     if (edited) {
       model.save();
-      logger.audit(
+      logger.info(
         { model: model.name },
         'model saved',
         'Model values updated due to dictionary publish',
@@ -160,7 +163,7 @@ export const publishDraft = async () => {
   return { dictionary, updatedModels };
 };
 
-export const countDraftStats = draft => {
+export const countDraftStats = (draft) => {
   const output = { edited: 0, new: 0 };
 
   const hasDependencies = draft.dependentValues && draft.dependentValues.length > 0;
@@ -202,12 +205,12 @@ export const editValue = (target, original, updated) => {
 };
 
 export const valueExists = (valuesList, value) => {
-  return !!valuesList.find(val => val.value === value);
+  return !!valuesList.find((val) => val.value === value);
 };
 
 export const removeValueIfNew = (valueList, value) => {
   const index = valueList.indexOf(
-    valueList.find(i => i.status === draftStatus.new && i.value === value),
+    valueList.find((i) => i.status === draftStatus.new && i.value === value),
   );
   if (index >= 0) {
     valueList.splice(index, 1);

cms/src/helpers/genomicVariants.js

@@ -10,7 +10,7 @@ import {
 import getLogger from '../logger.js';
 const logger = getLogger('helpers/genomicVariants');
 
-export const clearGenomicVariants = async name => {
+export const clearGenomicVariants = async (name) => {
   // Stop any active imports, if any:
   try {
     await VariantImporter.stopImport(name);
@@ -37,8 +37,8 @@ export const clearGenomicVariants = async name => {
   }
 };
 
-const titleCase = (text, filter = i => true) => {
-  return text.replace(/\w\S*/g, i => {
+const titleCase = (text, filter = (i) => true) => {
+  return text.replace(/\w\S*/g, (i) => {
     if (filter(i)) {
       return i.charAt(0).toUpperCase() + i.substr(1);
     }
@@ -68,9 +68,9 @@ const buildVariantId = ({
   }
 };
 
-const buildModelUrl = caseId => `${BASE_GDC_URL}/cases/${caseId}`;
-const buildMafUrl = fileId => `${BASE_GDC_URL}/files/${fileId}`;
-const buildSequenceUrl = caseId => `${BASE_GDC_URL}/cases/${caseId}#files`;
+const buildModelUrl = (caseId) => `${BASE_GDC_URL}/cases/${caseId}`;
+const buildMafUrl = (fileId) => `${BASE_GDC_URL}/files/${fileId}`;
+const buildSequenceUrl = (caseId) => `${BASE_GDC_URL}/cases/${caseId}#files`;
 
 export const addGenomicVariantsFromMaf = async (name, mafData, { filename, fileId }, caseId) => {
   const model = await Model.findOne({ name });
@@ -103,7 +103,7 @@ export const addGenomicVariantsFromMaf = async (name, mafData, { filename, fileI
 
       // Properties originally from Reference that are temporarily taken from MAF - Jon Eubank 2020-09-29
       const gene = row.Hugo_Symbol;
-      const gene_biotype = titleCase(row.BIOTYPE.replace(/_/g, ' '), i => !i.includes('RNA'));
+      const gene_biotype = titleCase(row.BIOTYPE.replace(/_/g, ' '), (i) => !i.includes('RNA'));
       const name = `${row.Hugo_Symbol} ${aa_change}`;
       const synonyms = [];
 
@@ -181,7 +181,7 @@ export const addGenomicVariantsFromMaf = async (name, mafData, { filename, fileI
     }
 
     await model.save();
-    logger.audit(
+    logger.info(
       {
         model: model.name,
         genomic_variants: model.genomic_variants ? model.genomic_variants.length : 0,