#1142 OpenSearch & AWS migration

.gitignore

@@ -18,12 +18,9 @@ node_modules
 .env.production.local
 .vscode/
 cms/env
-cms/variant-migrations/config.js
 
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
-docker/mongo
-docker/elasticsearch/data
-
+docker/

README.md

@@ -38,7 +38,7 @@ To run the required migrations:
 
 ```
 cd cms/variant-migrations
-../../node_modules/.bin/migrate-mongo up -f migrate-mongo-config.js
+../../node_modules/.bin/migrate-mongo up -f config.js
 ```
 
 ### Quickstart
@@ -59,7 +59,7 @@ yarn
 
 ```
 cd cms/variant-migrations
-../../node_modules/.bin/migrate-mongo up -f migrate-mongo-config.js
+../../node_modules/.bin/migrate-mongo up -f config.js
 ```
 
 4. Initialize ElasticSearch:

api/.env.schema

@@ -1,9 +1,11 @@
 ES_URL=
+ES_USER=
+ES_PASS=
 ES_HOST=
 PROJECT_ID=
 PORT=
 ES_UPDATE_INDEX=
 ENABLE_ADMIN=
-MONGODB_URI=mongodb://user@pass:localhost:27017/hcmi
-
 LOG_LEVEL=
+NODE_ENV=
+SEARCH_CLIENT_TYPE=

api/index.js

@@ -8,4 +8,4 @@ require('@babel/register')({
   ],
 });
 
-require('./src/index');
+require('./src/index.ts');

api/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@hcmi-portal/api",
   "version": "0.0.1",
-  "description": "``` npm i @overture-stack/arranger-server ```",
+  "description": "``` npm i @overture-stack/arranger-graphql-router ```",
   "private": true,
   "main": "index.js",
   "scripts": {
@@ -11,8 +11,11 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
-    "@elastic/elasticsearch": "~7.17.0",
-    "@overture-stack/arranger-server": "^3.0.0",
+    "@aws-sdk/credential-provider-node": "^3.972.66",
+    "@opensearch-project/opensearch": "^3.6.0",
+    "@overture-stack/arranger-graphql-router": "1.0.0-rc.4",
+    "@overture-stack/arranger-types": "1.0.0-rc.3",
+    "@overture-stack/sqon-builder": "^1.1.0",
     "JSONStream": "^1.3.5",
     "babel-polyfill": "^6.26.0",
     "dotenv": "^5.0.1",
@@ -21,7 +24,7 @@
     "helmet": "^3.15.0",
     "jszip": "^3.7.0",
     "map-stream": "^0.0.7",
-    "mongoose": "7.8.7",
+    "mongoose": "^8.23.0",
     "pino": "^9.13.1",
     "pino-http": "^10.5.0",
     "sharp": "0.34.4"

api/src/dataExport.js

@@ -1,12 +1,13 @@
 import express from 'express';
 import bodyParser from 'body-parser';
 import expressSanitizer from 'express-sanitizer';
-import { dataStream } from '@overture-stack/arranger-server/dist/download';
-import getAllData from '@overture-stack/arranger-server/dist/utils/getAllData';
+import { dataStream } from '@overture-stack/arranger-graphql-router/download';
+import { getAllData } from '@overture-stack/arranger-graphql-router/utils';
 import JSZip from 'jszip';
 import map from 'map-stream';
 import through2 from 'through2';
-import getLogger from './logger';
+
+import getLogger from './logger.js';
 
 const logger = getLogger('dataExport');
 
@@ -48,7 +49,11 @@ dataExportRouter.post('/models', async (req, res) => {
      * We can get a stream from arranger using the getAllData method and then can store the variant data for each model
      * for future processing.
      */
-    const allDataStream = await getAllData({ sqon, maxRows: 100, mock: {}, ctx: req.context });
+    const allDataStream = await getAllData({
+      sqon,
+      maxRows: 100,
+      ctx: req.context,
+    });
     const collectVariantData = through2.obj(function ({ hits }, enc, callback) {
       hits.forEach((model) => {
         if (model?.genomic_variants && model?.genomic_variants.length > 0) {

api/src/health.js

@@ -1,14 +1,10 @@
 import express from 'express';
-import elasticsearch from '@elastic/elasticsearch';
-
 import _ from 'lodash';
 
-const startTime = Date.now();
-
-const client = new elasticsearch.Client({
-  node: process.env.ES_URL,
-});
+import pm2 from './pm2.ts';
+import getClient from './services/searchClient.ts';
 
+const startTime = Date.now();
 const healthRouter = express.Router();
 
 healthRouter.get('/', async (req, res) => {
@@ -18,6 +14,7 @@ healthRouter.get('/', async (req, res) => {
 
 healthRouter.get('/es', async (req, res) => {
   try {
+    const client = getClient(pm2);
     const status = await client.ping();
     if (_.get(status, 'statusCode') === 200) {
       const response = _.omit(status, 'meta');

api/src/index.ts

@@ -0,0 +1,85 @@
+import 'babel-polyfill';
+import bodyParser from 'body-parser';
+import cors from 'cors';
+import express from 'express';
+import expressSanitizer from 'express-sanitizer';
+import helmet from 'helmet';
+import { Server } from 'http';
+import ArrangerRouter, { type ArrangerBaseContext } from '@overture-stack/arranger-graphql-router';
+import type { ConfigsObject, DisplayType, ExtendedConfigs, FacetsConfigs, MatchBoxConfigs, TableConfigs } from '@overture-stack/arranger-types/configs';
+import * as path from 'path';
+
+import baseConfig from '../../elasticsearch/arranger_metadata/base.json' with { type: 'json' };
+import extendedConfigFile from '../../elasticsearch/arranger_metadata/extended.json' with { type: 'json' };
+import facetsConfigFile from '../../elasticsearch/arranger_metadata/facets.json' with { type: 'json' };
+import matchboxConfigFile from '../../elasticsearch/arranger_metadata/matchbox.json' with { type: 'json' };
+import tableConfigFile from '../../elasticsearch/arranger_metadata/table.json' with { type: 'json' };
+
+import dataExportRouter from './dataExport.js';
+import getClient from './services/searchClient.ts';
+import getLogger from './logger.js';
+import healthRouter from './health.js';
+import lastUpdatedRouter from './lastUpdated.js';
+import pm2 from './pm2.ts';
+import searchRouter from './search.js';
+
+const logger = getLogger('root');
+const port = process.env.PORT || 5050;
+const app = express();
+const http = new Server(app);
+
+app.use(helmet());
+app.use(bodyParser.json());
+app.use(bodyParser.urlencoded({ extended: true }));
+app.use(expressSanitizer()); // each route is responsible for sanitization
+app.use(cors());
+
+// Swagger
+app.use('/docs', (req, res) => {
+  res.sendFile(path.join(__dirname, '../redoc.html'));
+});
+app.use('/swagger', (req, res) => {
+  res.sendFile(path.join(__dirname, '../swagger.json'));
+});
+
+// Arranger Configs
+const displayTypeValues: DisplayType[] = ['all' , 'bits' , 'boolean' , 'bytes' , 'date' , 'list' , 'nested' , 'number'];
+
+const extendedConfigs: ExtendedConfigs[] = extendedConfigFile['extended'].map(configRecord => {
+  const {type: stringType} = configRecord;
+  const displayType = displayTypeValues.find(type => stringType === type) || 'all';
+  return {...configRecord, type: displayType};
+});
+const facetConfigs: FacetsConfigs = facetsConfigFile['facets'];
+const matchboxConfigs: MatchBoxConfigs[] = matchboxConfigFile['matchbox'];
+const tableConfigs: TableConfigs = tableConfigFile['table'];
+
+const disableGraphQLIntrospection = process.env.DISABLE_GRAPHQL_INTROSPECTION === 'true' || pm2.DISABLE_GRAPHQL_INTROSPECTION || false;
+const esHost = process.env.ES_HOST || pm2.ES_URL || 'http://localhost:9200';
+
+const configs: Partial<ConfigsObject<ArrangerBaseContext>> = {
+  ...baseConfig,
+  disableGraphQLIntrospection,
+  esHost,
+  esIndex: 'hcmi',
+  extended: extendedConfigs,
+  enableDebug: true,
+  facets: facetConfigs,
+  matchbox: matchboxConfigs,
+  maxDepth: 10,
+  table: tableConfigs
+};
+
+const esClient = getClient(pm2);
+
+ArrangerRouter({ configs, esClient }).then((router) => {
+  app.use(router);
+  app.use('/last-updated', lastUpdatedRouter);
+  app.use('/health', healthRouter);
+  app.use('/search', searchRouter);
+  app.use('/export', dataExportRouter);
+
+  http.listen(port, async () => {
+    logger.info({ port }, 'API Started!');
+  });
+});

api/src/lastUpdated.js

@@ -1,14 +1,19 @@
 // @ts-check
 
 import express from 'express';
-import esClient from './services/elasticsearch';
+
+import pm2 from './pm2.ts';
+import getClient from './services/searchClient.ts';
+import getLogger from './logger.js';
 
 const lastUpdatedRouter = express.Router();
+const logger = getLogger('lastUpdated Router');
 
 lastUpdatedRouter.get('/', async (req, res) => {
+  const searchClient = getClient(pm2);
   try {
-    const response = await esClient.search({
-      index: process.env.ES_UPDATE_INDEX,
+    const response = await searchClient?.search({
+      index: process.env.ES_UPDATE_INDEX || pm2.ES_UPDATE_INDEX || 'hcmi-update',
       body: {
         query: {
           match_all: {},
@@ -23,8 +28,9 @@ lastUpdatedRouter.get('/', async (req, res) => {
         ],
       },
     });
-    return res.json(response.body.hits.hits[0]._source);
+    return res.json(response?.body?.hits?.hits[0]?._source);
   } catch (error) {
+    logger.error(`Error retrieving last updated date from Arranger: ${error}`);
     return res.status(500).json({
       error: error,
     });

api/src/pm2.ts

@@ -0,0 +1,21 @@
+import pm2Config from './../pm2.config.js';
+
+// PM2 Env Setup
+type pm2EnvValues = 'dev' | 'prd' | 'staging';
+
+const pm2EnvValues = ['dev', 'prd', 'staging'];
+const pm2Env: pm2EnvValues =
+  process.env.ENV && pm2EnvValues.includes(process.env.ENV)
+    ? (process.env.ENV as pm2EnvValues)
+    : 'dev';
+
+const pm2ConfigGeneric =
+  (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0].env) || {};
+const pm2ConfigForEnv =
+  (pm2Config && pm2Config.apps && pm2Config.apps[0] && pm2Config.apps[0][`env_${pm2Env}`]) || {};
+
+const pm2 = { ...pm2ConfigGeneric, ...pm2ConfigForEnv };
+
+export type PM2Config = typeof pm2;
+
+export default pm2;

api/src/search.js

@@ -1,6 +1,8 @@
 import express from 'express';
-import esClient from './services/elasticsearch';
-import { get } from 'lodash';
+import _ from 'lodash';
+
+import pm2 from './pm2.ts';
+import getClient from './services/searchClient.ts';
 
 const GENES_INDEX = 'genes';
 const VARIANTS_INDEX = 'genomic_variants';
@@ -28,11 +30,12 @@ geneSearchRouter.get('/gene', async (req, res) => {
       },
     };
 
-    const response = await esClient.search({
+    const searchClient = getClient(pm2);
+    const response = await searchClient.search({
       index: GENES_INDEX,
       body: { query },
     });
-    const genes = get(response, 'body.hits.hits', []).map((i) => i._source);
+    const genes = _.get(response, 'body.hits.hits', []).map((i) => i._source);
     res.status(200).json({ genes });
   } catch (err) {
     console.log('Failure performing gene search:', err);
@@ -57,11 +60,12 @@ geneSearchRouter.get('/variant', async (req, res) => {
       },
     };
 
-    const response = await esClient.search({
+    const searchClient = getClient(pm2);
+    const response = await searchClient.search({
       index: VARIANTS_INDEX,
       body: { query },
     });
-    const genes = get(response, 'body.hits.hits', []).map((i) => i._source);
+    const genes = _.get(response, 'body.hits.hits', []).map((i) => i._source);
     res.status(200).json({ genes });
   } catch (err) {
     console.log('Failure performing genomic variants search:', err);