Merge discvr-26.3 to develop

SequenceAnalysis/api-src/org/labkey/api/sequenceanalysis/pipeline/AlignerIndexUtil.java

@@ -5,6 +5,7 @@
 import org.labkey.api.data.ConvertHelper;
 import org.labkey.api.pipeline.PipelineJobException;
 import org.labkey.api.pipeline.WorkDirectory;
+import org.labkey.api.util.FileUtil;
 import org.labkey.vfs.FileSystemLike;
 
 import java.io.File;
@@ -85,20 +86,25 @@ private static boolean verifyOrCreateCachedIndex(PipelineContext ctx, @Nullable
                     if (wd != null)
                     {
                         String val = ctx.getJob().getParameters().get(COPY_LOCALLY);
-                        boolean doCopy = forceCopyLocal || (val == null || ConvertHelper.convert(val, Boolean.class));
+                        boolean doCopy = forceCopyLocal || (val == null || Boolean.TRUE.equals(ConvertHelper.convert(val, Boolean.class)));
 
                         if (doCopy)
                         {
                             ctx.getLogger().info("copying index files to work location");
-                            File localSharedDir = new File(wd.getDir().toNioPathForRead().toFile(), "Shared");
-                            File destination = new File(localSharedDir, localName);
+                            File localSharedDir = FileUtil.appendName(wd.getDir().toNioPathForRead().toFile(), "Shared");
+                            File destination = FileUtil.appendName(localSharedDir, localName);
                             ctx.getLogger().debug(destination.getPath());
                             File[] files = webserverIndexDir.listFiles();
                             if (files == null)
                             {
                                 return false;
                             }
 
+                            if (!destination.exists())
+                            {
+                                FileUtil.mkdirs(destination);
+                            }
+
                             destination = wd.inputFile(FileSystemLike.wrapFile(webserverIndexDir), FileSystemLike.wrapFile(destination), true).toNioPathForRead().toFile();
                             if (output != null && !destination.equals(webserverIndexDir))
                             {

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisMaintenanceTask.java

@@ -272,10 +272,10 @@ else if (sf.getFilePath() == null)
             return;
         }
 
-        File root = new File(sf.getFilePath());
+        File root = new File(sf.getFilePath()).getParentFile();
         if (!root.exists())
         {
-            log.error("Run fileroot does not exist: " + runId + " / " + root.getPath());
+            log.error("Run file root does not exist. runId: " + runId + " / jobId: " + sf.getRowId() + " / " + root.getPath());
             return;
         }
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -127,6 +127,7 @@
 import org.labkey.sequenceanalysis.run.analysis.SawfishJointCallingHandler;
 import org.labkey.sequenceanalysis.run.analysis.SequenceBasedTypingAnalysis;
 import org.labkey.sequenceanalysis.run.analysis.SnpCountAnalysis;
+import org.labkey.sequenceanalysis.run.analysis.SpecHlaAnalysis;
 import org.labkey.sequenceanalysis.run.analysis.SubreadAnalysis;
 import org.labkey.sequenceanalysis.run.analysis.UnmappedReadExportHandler;
 import org.labkey.sequenceanalysis.run.analysis.ViralAnalysis;
@@ -343,6 +344,7 @@ public static void registerPipelineSteps()
         SequencePipelineService.get().registerPipelineStep(new PindelAnalysis.Provider());
         SequencePipelineService.get().registerPipelineStep(new PbsvAnalysis.Provider());
         SequencePipelineService.get().registerPipelineStep(new GenrichStep.Provider());
+        SequencePipelineService.get().registerPipelineStep(new SpecHlaAnalysis.Provider());
         SequencePipelineService.get().registerPipelineStep(new SawfishAnalysis.Provider());
 
         SequencePipelineService.get().registerPipelineStep(new PARalyzerAnalysis.Provider());

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/TaskFileManagerImpl.java

@@ -32,6 +32,7 @@
 import org.labkey.api.util.FileType;
 import org.labkey.api.util.FileUtil;
 import org.labkey.api.util.Pair;
+import org.labkey.api.util.Path;
 import org.labkey.api.writer.PrintWriters;
 import org.labkey.sequenceanalysis.SequenceAnalysisManager;
 import org.labkey.sequenceanalysis.SequenceAnalysisSchema;
@@ -57,6 +58,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -250,7 +252,7 @@ private SequenceAnalysisJobSupport getSequenceSupport()
 
     private File getDeferredDeleteLog(boolean create)
     {
-        File logFile = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), "toDelete.txt");
+        File logFile = FileUtil.appendName(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), "toDelete.txt");
         if (create && !logFile.exists())
         {
             try
@@ -269,7 +271,7 @@ private File getDeferredDeleteLog(boolean create)
 
     private File getMetricsLog(boolean create)
     {
-        File logFile = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), "metricsToCreate.txt");
+        File logFile = FileUtil.appendName(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), "metricsToCreate.txt");
         if (create && !logFile.exists())
         {
             try (PrintWriter writer = PrintWriters.getPrintWriter(logFile))
@@ -385,10 +387,10 @@ private File convertRelPathToFile(String line)
             return null;
         }
 
-        File f = new File(_workLocation, line);
+        File f = FileUtil.appendPath(_workLocation, Path.parse(line));
         if (!f.exists())
         {
-            File test = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), line);
+            File test = FileUtil.appendPath(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), Path.parse(line));
             if (test.exists())
             {
                 f = test;
@@ -859,7 +861,7 @@ private void processCopiedFile(File original, File moved, Collection<RecordedAct
                 _job.getLogger().debug("Directory has " + moved.listFiles().length + " children");
                 for (File f : moved.listFiles())
                 {
-                    processCopiedFile(new File(original, f.getName()), f, actions, resumer);
+                    processCopiedFile(FileUtil.appendName(original, f.getName()), f, actions, resumer);
                 }
             }
         }
@@ -910,23 +912,21 @@ public void cleanup(Collection<RecordedAction> actions, @Nullable AbstractResume
                 _job.getLogger().debug("discarding copied inputs");
                 _wd.discardCopiedInputs();
 
-                if (!_wd.getDir().exists())
+                if (_wd.getDir().exists())
                 {
-                    throw new PipelineJobException("work dir does not exist: " + _wd.getDir());
-                }
-
-                //NOTE: preserving relative locations is a pain.  therefore we copy all outputs, including directories
-                //then sort out which files were specified as named outputs later
-                for (File input : _wd.getDir().toNioPathForRead().toFile().listFiles())
-                {
-                    if (input.getName().matches("^core.[0-9]+$") || input.getName().endsWith(".hprof"))
+                    //NOTE: preserving relative locations is a pain.  therefore we copy all outputs, including directories
+                    //then sort out which files were specified as named outputs later
+                    for (File input : Objects.requireNonNull(_wd.getDir().toNioPathForRead().toFile().listFiles()))
                     {
-                        _job.getLogger().debug("Deleting core/hprof file: " + input.getPath());
-                        input.delete();
-                        continue;
-                    }
+                        if (input.getName().matches("^core.[0-9]+$") || input.getName().endsWith(".hprof"))
+                        {
+                            _job.getLogger().debug("Deleting core/hprof file: " + input.getPath());
+                            input.delete();
+                            continue;
+                        }
 
-                    copyFile(input, actions, resumer);
+                        copyFile(input, actions, resumer);
+                    }
                 }
             }
             else
@@ -987,7 +987,7 @@ private void doCopyFile(File input, Collection<RecordedAction> actions, @Nullabl
         }
 
         String path = _wd.getRelativePath(FileSystemLike.wrapFile(input));
-        File dest = new File(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), path);
+        File dest = FileUtil.appendPath(getSupport().getAnalysisDirectory().toNioPathForRead().toFile(), Path.parse(path));
         _job.getLogger().debug("to: " + dest.getPath());
 
         boolean doMove = true;
@@ -1076,7 +1076,7 @@ private File decompressFile(File i, List<RecordedAction> actions)
             //NOTE: we use relative paths in all cases here
             _job.getLogger().info("Decompressing file: " + i.getPath());
 
-            unzipped = new File(_wd.getDir().toNioPathForRead().toFile(), i.getName().replaceAll(".gz$", ""));
+            unzipped = FileUtil.appendName(_wd.getDir().toNioPathForRead().toFile(), i.getName().replaceAll(".gz$", ""));
             unzipped = Compress.decompressGzip(i, unzipped);
             _job.getLogger().debug("\tunzipped: " + unzipped.getPath());
 

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/analysis/SpecHlaAnalysis.java

@@ -0,0 +1,180 @@
+package org.labkey.sequenceanalysis.run.analysis;
+
+import htsjdk.samtools.SAMFileHeader;
+import org.apache.commons.io.FileUtils;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.model.AnalysisModel;
+import org.labkey.api.sequenceanalysis.model.Readset;
+import org.labkey.api.sequenceanalysis.pipeline.AbstractAnalysisStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.AnalysisOutputImpl;
+import org.labkey.api.sequenceanalysis.pipeline.AnalysisStep;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineContext;
+import org.labkey.api.sequenceanalysis.pipeline.PipelineStepProvider;
+import org.labkey.api.sequenceanalysis.pipeline.ReferenceGenome;
+import org.labkey.api.sequenceanalysis.pipeline.SamSorter;
+import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
+import org.labkey.api.sequenceanalysis.run.AbstractCommandPipelineStep;
+import org.labkey.api.sequenceanalysis.run.AbstractCommandWrapper;
+import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
+import org.labkey.api.util.FileUtil;
+import org.labkey.api.util.Path;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class SpecHlaAnalysis extends AbstractCommandPipelineStep<SimpleScriptWrapper> implements AnalysisStep
+{
+    public SpecHlaAnalysis(PipelineStepProvider<?> provider, PipelineContext ctx)
+    {
+        super(provider, ctx, new SimpleScriptWrapper(ctx.getLogger()));
+    }
+
+    public static class Provider extends AbstractAnalysisStepProvider<SpecHlaAnalysis>
+    {
+        public Provider()
+        {
+            super("SpecHlaStep", "SpecHLA", null, "This will run SpecHLA for HLA genotyping from WGS/WXS data. This should use a BAM aligned to a custom HLA DB, rather than aligned to the full genome", Arrays.asList(
+
+            ), null, "https://github.com/deepomicslab/SpecHLA/");
+        }
+
+        @Override
+        public SpecHlaAnalysis create(PipelineContext ctx)
+        {
+            return new SpecHlaAnalysis(this, ctx);
+        }
+    }
+
+    @Override
+    public Output performAnalysisPerSampleRemote(Readset rs, File inputBam, ReferenceGenome referenceGenome, File outputDir) throws PipelineJobException
+    {
+        AnalysisOutputImpl output = new AnalysisOutputImpl();
+
+        File gzippedFasta = referenceGenome.getWorkingFastaFileGzipped();
+        if (!gzippedFasta.exists())
+        {
+            throw new PipelineJobException("Missing file: " + gzippedFasta.getPath());
+        }
+
+        File doneFile = FileUtil.appendName(outputDir, FileUtil.getBaseName(inputBam) + ".subset.done");
+        output.addIntermediateFile(doneFile);
+
+        File subsetBam = FileUtil.appendName(outputDir, FileUtil.getBaseName(inputBam) + ".subset.bam");
+        SamtoolsRunner sr = new SamtoolsRunner(getWrapper().getLogger());
+        if (doneFile.exists())
+        {
+            getPipelineCtx().getLogger().debug("Done file exists, skipping samtools view");    
+        }
+        else
+        {
+            sr.execute(Arrays.asList(
+                    sr.getSamtoolsPath().getPath(),
+                    "view",
+                    "-h",
+                    "-F", "12", //This selects pairs where either mate is mapped
+                    "-T", gzippedFasta.getPath(),
+                    "-o", subsetBam.getPath(),
+                    inputBam.getPath()
+            ));
+        }
+        output.addIntermediateFile(subsetBam);
+
+        File queryNameSortBam = FileUtil.appendName(outputDir, FileUtil.getBaseName(inputBam) + ".querySort.bam");
+        if (doneFile.exists())
+        {
+            getPipelineCtx().getLogger().debug("Done file exists, skipping samtools sort");
+        }
+        else
+        {
+            new SamSorter(getPipelineCtx().getLogger()).execute(subsetBam, queryNameSortBam, SAMFileHeader.SortOrder.queryname);
+        }
+        output.addIntermediateFile(queryNameSortBam);
+
+        File fq1 = FileUtil.appendName(outputDir, FileUtil.getBaseName(inputBam) + ".R1.fastq.gz");
+        File fq2 = FileUtil.appendName(outputDir, FileUtil.getBaseName(inputBam) + ".R2.fastq.gz");
+        if (doneFile.exists())
+        {
+            getPipelineCtx().getLogger().debug("Done file exists, skipping samtools fastq");
+        }
+        else
+        {
+            sr.execute(Arrays.asList(
+                    sr.getSamtoolsPath().getPath(),
+                    "fastq",
+                    "-1",
+                    fq1.getPath(),
+                    "-2",
+                    fq2.getPath(),
+                    queryNameSortBam.getPath()
+            ));
+        }
+        output.addIntermediateFile(fq1);
+        output.addIntermediateFile(fq2);
+
+        try
+        {
+            FileUtils.touch(doneFile);
+        }
+        catch (IOException e)
+        {
+            throw new PipelineJobException(e);
+        }
+
+        File specHlaExe = AbstractCommandWrapper.resolveFileInPath("spechla", null, true);
+
+        List<String> toRun = new ArrayList<>(Arrays.asList(
+            specHlaExe.getPath(),
+            "-n",
+            "specHLA",
+            "-u",
+            "1", // 1 = exon. 0 = full-length
+            "-1",
+            fq1.getPath(),
+            "-2",
+            fq2.getPath(),
+            "-o",
+            outputDir.getPath()
+        ));
+
+        Integer maxThreads = SequencePipelineService.get().getMaxThreads(getWrapper().getLogger());
+        if (maxThreads != null)
+        {
+            toRun.add("-j");
+            toRun.add(maxThreads.toString());
+        }
+
+        getWrapper().execute(toRun);
+
+        File spechlaDir = FileUtil.appendName(outputDir, "specHLA");
+        File outFile = FileUtil.appendName(spechlaDir, "hla.result.txt");
+        if (!outFile.exists())
+        {
+            throw new PipelineJobException("SpecHLA result file does not exist: " + outFile.getPath());
+        }
+
+        output.addIntermediateFile(spechlaDir);
+        File copiedFile = FileUtil.appendName(outputDir, outFile.getName());
+        try
+        {
+            FileUtils.copyFile(outFile, copiedFile);
+        }
+        catch (IOException e)
+        {
+            throw new PipelineJobException(e);
+        }
+
+        output.addSequenceOutput(copiedFile, FileUtil.getBaseName(inputBam) + ": HLA Typing", "specHLA Genotyping", rs.getReadsetId(), null, referenceGenome.getGenomeId(), null);
+
+        return output;
+    }
+
+    @Override
+    public Output performAnalysisPerSampleLocal(AnalysisModel model, File inputBam, File referenceFasta, File outDir) throws PipelineJobException
+    {
+        return null;
+    }
+}