AUS419 purple ffpe

bam-tools/src/main/java/com/hartwig/hmftools/bamtools/metrics/GeneCoverage.java

@@ -5,6 +5,7 @@
 import static com.hartwig.hmftools.bamtools.common.CommonUtils.BT_LOGGER;
 import static com.hartwig.hmftools.common.driver.panel.DriverGenePanelConfig.DRIVER_GENE_PANEL;
 import static com.hartwig.hmftools.common.driver.panel.DriverGeneRegions.buildDriverGeneRegions;
+import static com.hartwig.hmftools.common.driver.panel.DriverGeneRegions.findInvalidDriverGenes;
 import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache.ENSEMBL_DATA_DIR;
 import static com.hartwig.hmftools.common.metrics.GeneDepthFile.generateExonCoverageFilename;
 import static com.hartwig.hmftools.common.metrics.GeneDepthFile.generateGeneCoverageFilename;
@@ -24,6 +25,7 @@
 import com.google.common.collect.Maps;
 import com.hartwig.hmftools.common.driver.panel.DriverGene;
 import com.hartwig.hmftools.common.driver.panel.DriverGenePanelConfig;
+import com.hartwig.hmftools.common.driver.panel.DriverGeneRegions;
 import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache;
 import com.hartwig.hmftools.common.gene.GeneRegion;
 import com.hartwig.hmftools.common.region.ChrBaseRegion;
@@ -57,6 +59,14 @@ public GeneCoverage(final ConfigBuilder configBuilder)
 
         List<DriverGene> driverGenes = DriverGenePanelConfig.loadDriverGenes(configBuilder);
 
+        List<DriverGene> invalidDriverGenes = findInvalidDriverGenes(driverGenes, ensemblDataCache);
+        if(!invalidDriverGenes.isEmpty())
+        {
+            BT_LOGGER.error("Invalid non-Ensembl gene names: {}",
+                    invalidDriverGenes.stream().map( x -> x.gene()).collect(Collectors.joining(";")));
+            System.exit(1);
+        }
+
         List<String> coverageGenes = driverGenes.stream().map(x -> x.gene()).collect(Collectors.toList());
 
         Map<String,List<GeneRegion>> chrGeneRegions = buildDriverGeneRegions(ensemblDataCache, coverageGenes);

gene-utils/src/main/java/com/hartwig/hmftools/geneutils/ensembl/EnsemblDAO.java

@@ -1,10 +1,6 @@
 package com.hartwig.hmftools.geneutils.ensembl;
 
 import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader.ENSEMBL_DELIM;
-import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader.ENSEMBL_GENE_DATA_FILE;
-import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader.ENSEMBL_PROTEIN_FEATURE_DATA_FILE;
-import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader.ENSEMBL_TRANS_EXON_DATA_FILE;
-import static com.hartwig.hmftools.common.gene.TranscriptUtils.codingBaseLength;
 import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeVersion.V38;
 import static com.hartwig.hmftools.common.genome.region.Strand.POS_STRAND;
 import static com.hartwig.hmftools.common.utils.file.FileWriterUtils.createBufferedWriter;
@@ -25,7 +21,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
@@ -64,9 +59,6 @@ public class EnsemblDAO
     // reference transcripts are loaded only to evaluate difference between Ensembl versions and v37 vs v38
     private final Map<String,List<TranscriptData>> mReferenceTranscriptMap; // keyed by geneId
 
-    private static final int COORD_SYSTEM_V37 = 2;
-    private static final int COORD_SYSTEM_V38 = 4;
-
     // not in HGNC but retained
     private static final List<GeneData> GENE_DATA_OVERRIDES = Lists.newArrayList(
             new GeneData("ENSG00000258414", "AL121790.1","14", POS_STRAND,
@@ -248,9 +240,9 @@ public void writeDataCacheFiles(final String outputDir)
         if (!outputFilename.endsWith(File.separator))
             outputFilename += File.separator;
 
-        writeGeneData(outputFilename + ENSEMBL_GENE_DATA_FILE);
-        writeTranscriptExonData(outputFilename + ENSEMBL_TRANS_EXON_DATA_FILE);
-        writeTranscriptProteinData(outputFilename + ENSEMBL_PROTEIN_FEATURE_DATA_FILE);
+        writeGeneData(outputFilename + EnsemblDataLoader.ensemblGeneDataFile(mRefGenomeVersion));
+        writeTranscriptExonData(outputFilename + EnsemblDataLoader.ensemblTransExonDataFile(mRefGenomeVersion));
+        writeTranscriptProteinData(outputFilename + EnsemblDataLoader.ensemblProteinDataFile(mRefGenomeVersion));
     }
 
     private void writeGeneData(final String outputFile)

gene-utils/src/main/java/com/hartwig/hmftools/geneutils/ensembl/GenerateEnsemblDataCache.java

@@ -1,6 +1,5 @@
 package com.hartwig.hmftools.geneutils.ensembl;
 
-import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader.ENSEMBL_TRANS_SPLICE_DATA_FILE;
 import static com.hartwig.hmftools.common.fusion.FusionCommon.DEFAULT_PRE_GENE_PROMOTOR_DISTANCE;
 import static com.hartwig.hmftools.common.fusion.FusionCommon.NEG_STRAND;
 import static com.hartwig.hmftools.common.fusion.FusionCommon.POS_STRAND;
@@ -21,6 +20,7 @@
 
 import com.google.common.collect.Lists;
 import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache;
+import com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader;
 import com.hartwig.hmftools.common.gene.GeneData;
 import com.hartwig.hmftools.common.gene.ExonData;
 import com.hartwig.hmftools.common.gene.TranscriptData;
@@ -63,19 +63,20 @@ public static void writeEnsemblDataFiles(final ConfigBuilder configBuilder)
         geneTransCache.load(false);
 
         createTranscriptPreGenePositionData(
-                geneTransCache.getChrGeneDataMap(), geneTransCache.getTranscriptDataMap(), DEFAULT_PRE_GENE_PROMOTOR_DISTANCE, outputDir);
+                geneTransCache.getChrGeneDataMap(), geneTransCache.getTranscriptDataMap(),
+                DEFAULT_PRE_GENE_PROMOTOR_DISTANCE, refGenomeVersion, outputDir);
 
         GU_LOGGER.info("Ensembl data cache complete");
     }
 
     private static void createTranscriptPreGenePositionData(
             final Map<String, List<GeneData>> chrGeneDataMap, final Map<String, List<TranscriptData>> transcriptDataMap,
-            int preGenePromotorDistance, final String outputDir)
+            int preGenePromotorDistance, final RefGenomeVersion refGenomeVersion, final String outputDir)
     {
         // generate a cache file of the nearest upstream splice acceptor from another gene for each transcript
         try
         {
-            String outputFile = outputDir + ENSEMBL_TRANS_SPLICE_DATA_FILE;
+            String outputFile = outputDir + EnsemblDataLoader.ensemblSpliceDataFile(refGenomeVersion);
 
             BufferedWriter writer = createBufferedWriter(outputFile, false);
 
@@ -85,13 +86,13 @@ private static void createTranscriptPreGenePositionData(
             // for each gene, collect up any gene which overlaps it or is within the specified distance upstream from it
             for(Map.Entry<String, List<GeneData>> entry : chrGeneDataMap.entrySet())
             {
-                final String chromosome = entry.getKey();
+                String chromosome = entry.getKey();
 
                 GU_LOGGER.debug("calculating pre-gene positions for chromosome({})", chromosome);
 
-                final List<GeneData> geneList = entry.getValue();
+                List<GeneData> geneList = entry.getValue();
 
-                for(final GeneData gene : geneList)
+                for(GeneData gene : geneList)
                 {
                     List<String> proximateGenes = Lists.newArrayList();
 
@@ -109,7 +110,7 @@ private static void createTranscriptPreGenePositionData(
                         geneRangeEnd = gene.GeneEnd + preGenePromotorDistance;
                     }
 
-                    for(final GeneData otherGene : geneList)
+                    for(GeneData otherGene : geneList)
                     {
                         if(otherGene.Strand != gene.Strand)
                             continue;
@@ -127,12 +128,12 @@ private static void createTranscriptPreGenePositionData(
                         continue;
 
                     // now set the preceding splice acceptor position for each transcript in this gene
-                    final List<TranscriptData> transDataList = transcriptDataMap.get(gene.GeneId);
+                    List<TranscriptData> transDataList = transcriptDataMap.get(gene.GeneId);
 
                     if(transDataList == null || transDataList.isEmpty())
                         continue;
 
-                    for(final TranscriptData transData : transDataList)
+                    for(TranscriptData transData : transDataList)
                     {
                         long transStartPos = gene.Strand == POS_STRAND ? transData.TransStart : transData.TransEnd;
 
@@ -167,14 +168,14 @@ private static long findFirstSpliceAcceptor(
     {
         long closestPosition = -1;
 
-        for(final String geneId : proximateGenes)
+        for(String geneId : proximateGenes)
         {
-            final List<TranscriptData> transDataList = transDataMap.get(geneId);
+            List<TranscriptData> transDataList = transDataMap.get(geneId);
 
             if(transDataList == null)
                 continue;
 
-            for(final TranscriptData transData : transDataList)
+            for(TranscriptData transData : transDataList)
             {
                 if(transId == transData.TransId)
                     continue;

gene-utils/src/main/java/com/hartwig/hmftools/geneutils/ensembl/ProteomeWriter.java

@@ -4,7 +4,6 @@
 import static java.lang.Math.min;
 
 import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache.addEnsemblDir;
-import static com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader.ENSEMBL_TRANS_AMINO_ACIDS_FILE;
 import static com.hartwig.hmftools.common.fusion.FusionCommon.POS_STRAND;
 import static com.hartwig.hmftools.common.gene.TranscriptUtils.calcCodingStartPositionAdjustment;
 import static com.hartwig.hmftools.common.genome.refgenome.RefGenomeSource.REF_GENOME;
@@ -27,6 +26,7 @@
 import com.hartwig.hmftools.common.codon.Codons;
 import com.hartwig.hmftools.common.codon.Nucleotides;
 import com.hartwig.hmftools.common.ensemblcache.EnsemblDataCache;
+import com.hartwig.hmftools.common.ensemblcache.EnsemblDataLoader;
 import com.hartwig.hmftools.common.gene.ExonData;
 import com.hartwig.hmftools.common.gene.GeneData;
 import com.hartwig.hmftools.common.gene.TranscriptAminoAcids;
@@ -67,7 +67,7 @@ public ProteomeWriter(final ConfigBuilder configBuilder)
 
         mRefGenome = loadRefGenome(configBuilder.getValue(REF_GENOME));
 
-        mWriter = initialiseWriter(parseOutputDir(configBuilder));
+        mWriter = initialiseWriter(parseOutputDir(configBuilder), mRefGenomeVersion);
     }
 
     public void run()
@@ -178,11 +178,11 @@ private void processTranscript(final GeneData geneData, final TranscriptData tra
         writeData(geneData.GeneId, geneData.GeneName, transData.TransName, transData.IsCanonical, aminoAcids);
     }
 
-    private static BufferedWriter initialiseWriter(final String outputDir)
+    private static BufferedWriter initialiseWriter(final String outputDir, final RefGenomeVersion refGenomeVersion)
     {
         try
         {
-            String filename = outputDir + ENSEMBL_TRANS_AMINO_ACIDS_FILE;
+            String filename = outputDir + EnsemblDataLoader.ensemblAminoAcidDataFile(refGenomeVersion);
             BufferedWriter writer = createBufferedWriter(filename, false);
 
             writer.write(TranscriptAminoAcids.csvHeader());

hmf-common/src/main/java/com/hartwig/hmftools/common/driver/panel/DriverGeneRegions.java

@@ -23,6 +23,11 @@
 
 public final class DriverGeneRegions
 {
+    public static List<DriverGene> findInvalidDriverGenes(final List<DriverGene> driverGenes, final EnsemblDataCache ensemblDataCache)
+    {
+        return driverGenes.stream().filter(x -> ensemblDataCache.getGeneDataByName(x.gene()) == null).collect(Collectors.toList());
+    }
+
     public static Map<String,List<GeneRegion>> buildDriverGeneRegions(
             final EnsemblDataCache ensemblDataCache, final List<String> driverGenes)
     {

hmf-common/src/main/java/com/hartwig/hmftools/common/ensemblcache/EnsemblDataCache.java

@@ -405,17 +405,23 @@ public boolean load(boolean delayTranscriptLoading)
         if(!delayTranscriptLoading)
         {
             if(!EnsemblDataLoader.loadTranscriptData(
-                    mDataPath, mTranscriptByGeneIdMap, mRestrictedGeneIdList, mRequireExons, mCanonicalTranscriptsOnly,
+                    mDataPath, mRefGenomeVersion, mTranscriptByGeneIdMap, mRestrictedGeneIdList, mRequireExons, mCanonicalTranscriptsOnly,
                     mRequireNonEnsemblTranscripts, Collections.emptyList()))
             {
                 return false;
             }
 
-            if(mRequireProteinDomains && !loadTranscriptProteinData(mDataPath, mEnsemblProteinDataMap, Sets.newHashSet()))
-                return false;
+            if(mRequireProteinDomains)
+            {
+                if(!loadTranscriptProteinData(mDataPath, mRefGenomeVersion, mEnsemblProteinDataMap, Sets.newHashSet()))
+                    return false;
+            }
 
-            if(mRequireSplicePositions && !loadTranscriptSpliceAcceptorData(mDataPath, mTransSpliceAcceptorPosDataMap, Sets.newHashSet()))
-                return false;
+            if(mRequireSplicePositions)
+            {
+                if(!loadTranscriptSpliceAcceptorData(mDataPath, mRefGenomeVersion, mTransSpliceAcceptorPosDataMap, Sets.newHashSet()))
+                    return false;
+            }
         }
 
         return true;
@@ -429,7 +435,7 @@ public boolean loadTranscriptData(final List<String> restrictedGeneIds)
     public boolean loadTranscriptData(final List<String> restrictedGeneIds, final List<String> nonCanonicalTrans)
     {
         if(!EnsemblDataLoader.loadTranscriptData(
-                mDataPath, mTranscriptByGeneIdMap, restrictedGeneIds, mRequireExons, mCanonicalTranscriptsOnly,
+                mDataPath, mRefGenomeVersion, mTranscriptByGeneIdMap, restrictedGeneIds, mRequireExons, mCanonicalTranscriptsOnly,
                 mRequireNonEnsemblTranscripts, nonCanonicalTrans))
         {
             return false;
@@ -442,10 +448,10 @@ public boolean loadTranscriptData(final List<String> restrictedGeneIds, final Li
             transDataList.forEach(x -> uniqueTransIds.add(x.TransId));
         }
 
-        if(mRequireProteinDomains && !loadTranscriptProteinData(mDataPath, mEnsemblProteinDataMap, uniqueTransIds))
+        if(mRequireProteinDomains && !loadTranscriptProteinData(mDataPath, mRefGenomeVersion, mEnsemblProteinDataMap, uniqueTransIds))
             return false;
 
-        if(mRequireSplicePositions && !loadTranscriptSpliceAcceptorData(mDataPath, mTransSpliceAcceptorPosDataMap, uniqueTransIds))
+        if(mRequireSplicePositions && !loadTranscriptSpliceAcceptorData(mDataPath, mRefGenomeVersion, mTransSpliceAcceptorPosDataMap, uniqueTransIds))
             return false;
 
         return true;