Molecule and PS bulk import by file

api/src/org/labkey/api/dataiterator/DataClassDataIteratorTransformer.java

@@ -0,0 +1,31 @@
+package org.labkey.api.dataiterator;
+
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Map;
+
+/**
+ * Extension point for DataClass-specific transformations in the pre-trigger DataIterator pipeline.
+ * Registered per DataClass name via {@link org.labkey.api.exp.api.ExperimentService#registerDataClassDataIteratorTransformer}.
+ * A fresh instance is created for each import operation since implementations may be stateful
+ * between {@link #prepareTranslator} and {@link #wrapDataIterator}.
+ */
+public interface DataClassDataIteratorTransformer
+{
+    /**
+     * Called during SimpleTranslator setup to inspect input columns and add placeholder columns
+     * (e.g., via {@link SimpleTranslator#addNullColumn}) that will be populated by the wrapping DataIterator.
+     *
+     * @return true if the transformer is active and {@link #wrapDataIterator} should be called
+     */
+    boolean prepareTranslator(@NotNull SimpleTranslator step0,
+                              @NotNull Map<String, Integer> inputColumnNameMap,
+                              @NotNull DataIteratorContext context);
+
+    /**
+     * Wraps the DataIterator to apply DataClass-specific transformations.
+     * Called after the pre-trigger pipeline is assembled, only if {@link #prepareTranslator} returned true.
+     */
+    @NotNull
+    DataIterator wrapDataIterator(@NotNull DataIterator input, @NotNull DataIteratorContext context);
+}

api/src/org/labkey/api/exp/api/ExperimentService.java

@@ -32,6 +32,7 @@
 import org.labkey.api.data.RemapCache;
 import org.labkey.api.data.SQLFragment;
 import org.labkey.api.data.TableInfo;
+import org.labkey.api.dataiterator.DataClassDataIteratorTransformer;
 import org.labkey.api.exp.ExperimentDataHandler;
 import org.labkey.api.exp.ExperimentException;
 import org.labkey.api.exp.ExperimentProtocolHandler;
@@ -74,6 +75,7 @@
 import org.labkey.api.pipeline.RecordedActionSet;
 import org.labkey.api.query.BatchValidationException;
 import org.labkey.api.query.FilteredTable;
+import org.labkey.api.query.QueryUpdateService;
 import org.labkey.api.query.QueryKey;
 import org.labkey.api.query.QueryViewProvider;
 import org.labkey.api.query.UserSchema;
@@ -101,6 +103,8 @@
 import java.util.Set;
 import java.util.concurrent.locks.Lock;
 import java.util.function.Predicate;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
 
 import static org.labkey.api.exp.api.ExpDataClass.NEW_DATA_CLASS_ALIAS_VALUE;
 import static org.labkey.api.exp.api.SampleTypeService.NEW_SAMPLE_TYPE_ALIAS_VALUE;
@@ -657,6 +661,21 @@ static void validateParentAlias(Map<String, String> aliasMap, Set<String> reserv
 
     ExpDataClassDataTable createDataClassDataTable(String name, UserSchema schema, ContainerFilter cf, @NotNull ExpDataClass dataClass);
 
+    /**
+     * Registers a factory that creates a {@link DataClassDataIteratorTransformer}
+     * for the specified DataClass name. The transformer is applied in the pre-trigger DataIterator pipeline,
+     * allowing modules to add computed columns (e.g., transforming flat columns into JSON) that work
+     * uniformly for file imports, API imports, folder imports, and background pipeline jobs.
+     * A fresh instance is created per import via the factory since transformers may be stateful.
+     */
+    void registerDataClassDataIteratorTransformer(String dataClassName, @NotNull Supplier<DataClassDataIteratorTransformer> factory);
+
+    /**
+     * Returns a fresh {@link DataClassDataIteratorTransformer} for the given
+     * DataClass name, or {@code null} if none is registered.
+     */
+    @Nullable DataClassDataIteratorTransformer getDataClassDataIteratorTransformer(String dataClassName);
+
     ExpProtocolTable createProtocolTable(String name, UserSchema schema, ContainerFilter cf);
 
     ExpExperimentTable createExperimentTable(String name, UserSchema schema, ContainerFilter cf);

api/src/org/labkey/api/pipeline/AppPipelineJobNotificationProvider.java

@@ -31,6 +31,7 @@ abstract public class AppPipelineJobNotificationProvider implements PipelineJobN
     public enum ImportType {
         samples,
         sources,
+        registry,
         assays;
 
         public static ImportType getImportType(PipelineJob job)
@@ -41,7 +42,11 @@ public static ImportType getImportType(PipelineJob job)
                 if (schemaName.equalsIgnoreCase("samples"))
                     return samples;
                 else if (schemaName.equalsIgnoreCase("exp.data"))
+                {
+                    if ("Biologics".equalsIgnoreCase(queryImportPipelineJob.getImportContextBuilder().getJobNotificationProvider()))
+                        return registry;
                     return sources;
+                }
                 else if (schemaName.equalsIgnoreCase("exp"))
                 {
                     String queryName = queryImportPipelineJob.getImportContextBuilder().getQueryName();
@@ -125,29 +130,8 @@ public URLHelper getPipelineStatusHref(PipelineJob job)
             if (importType == null)
                 return null;
 
-            String urlFragment = "/" + importType.name();
-
-            ActionURL appURL = getAppURL(job.getContainer());
-
-            String type = queryImportPipelineJob.getImportContextBuilder().getQueryName();
-            if (!"materials".equalsIgnoreCase(type) || !"exp".equalsIgnoreCase(queryImportPipelineJob.getImportContextBuilder().getSchemaName()))
-                urlFragment += "/" + type + "?";
-            else
-                urlFragment += "?";
-
-            String and = "";
-
-            if (queryImportPipelineJob.getTransactionAuditId() > 0)
-            {
-                urlFragment = urlFragment + "transactionAuditId=" + queryImportPipelineJob.getTransactionAuditId();
-                and = "&";
-            }
-
-            String filename = queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName();
-            if (!StringUtils.isEmpty(filename))
-                urlFragment = urlFragment + and + "importFile=" + PageFlowUtil.encode(filename);
-
-            return appURL.setFragment(urlFragment);
+            String urlFragment = buildQueryImportUrlFragment(queryImportPipelineJob, importType, queryImportPipelineJob.getAdditionalJobResponseInfo());
+            return getAppURL(job.getContainer()).setFragment(urlFragment);
         }
 
         return null;
@@ -240,36 +224,58 @@ else if (job instanceof AssayUploadPipelineJob<?> assayJob)
         return null;
     }
 
-    private String getJobSuccessUrl(PipelineJob job, @NotNull ImportType importType, @Nullable Map<String, Object> info)
+    private String buildQueryImportUrlFragment(QueryImportPipelineJob queryImportPipelineJob, ImportType importType, @Nullable Map<String, Object> info)
     {
-        ActionURL appURL = getAppURL(job.getContainer());
         String urlFragment = "/" + importType.name();
-        if (job instanceof QueryImportPipelineJob queryImportPipelineJob)
+
+        Boolean isCrossType = queryImportPipelineJob.getImportContextBuilder().getOptionParamsMap().get(AbstractQueryImportAction.Params.crossTypeImport);
+        if (Boolean.TRUE.equals(isCrossType))
+            urlFragment = "/crossType/" + importType.name() + "?";
+        else if (info != null && info.containsKey("viewJobDataUrl"))
+        {
+            urlFragment = info.get("viewJobDataUrl").toString();
+            if (!urlFragment.endsWith("?") && !urlFragment.endsWith("&"))
+                urlFragment += "?";
+        }
+        else
         {
-            Boolean isCrossType = queryImportPipelineJob.getImportContextBuilder().getOptionParamsMap().get(AbstractQueryImportAction.Params.crossTypeImport);
-            if (isCrossType)
-                urlFragment = "/crossType/" + importType.name() + "?";
+            String type = queryImportPipelineJob.getImportContextBuilder().getQueryName();
+            String schemaName = queryImportPipelineJob.getImportContextBuilder().getSchemaName();
+            if ("materials".equalsIgnoreCase(type) && "exp".equalsIgnoreCase(schemaName))
+                urlFragment += "?";
             else
-            {
-                String type = queryImportPipelineJob.getImportContextBuilder().getQueryName();
                 urlFragment += "/" + PageFlowUtil.encode(type) + "?";
-            }
+        }
 
-            String and = "";
-            if (info != null)
+        String and = "";
+        if (info != null)
+        {
+            Long transactionAuditId = asLong(info.get("transactionAuditId"));
+            if (transactionAuditId != null && transactionAuditId > 0)
             {
-                Long transactionAuditId = (Long) info.get("transactionAuditId");
-                urlFragment = urlFragment + "transactionAuditId=" + transactionAuditId ;
+                urlFragment = urlFragment + "transactionAuditId=" + transactionAuditId;
                 and = "&";
             }
+        }
 
-            String filename = queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName();
-            if (!StringUtils.isEmpty(filename))
-                urlFragment = urlFragment + and + "importFile=" + PageFlowUtil.encode(filename);
+        String filename = queryImportPipelineJob.getImportContextBuilder().getPrimaryFile().getName();
+        if (!StringUtils.isEmpty(filename))
+            urlFragment = urlFragment + and + "importFile=" + PageFlowUtil.encode(filename);
 
+        return urlFragment;
+    }
+
+    private String getJobSuccessUrl(PipelineJob job, @NotNull ImportType importType, @Nullable Map<String, Object> info)
+    {
+        ActionURL appURL = getAppURL(job.getContainer());
+        String urlFragment;
+        if (job instanceof QueryImportPipelineJob queryImportPipelineJob)
+        {
+            urlFragment = buildQueryImportUrlFragment(queryImportPipelineJob, importType, info);
         }
         else if (job instanceof AssayUploadPipelineJob)
         {
+            urlFragment = "/" + importType.name();
             if (info != null)
             {
                 String provider = (String) info.get("provider");
@@ -278,6 +284,10 @@ else if (job instanceof AssayUploadPipelineJob)
                 urlFragment = urlFragment + "/" + PageFlowUtil.encode(provider) + "/" + PageFlowUtil.encode(assayName) + "/runs/" + runId;
             }
         }
+        else
+        {
+            urlFragment = "/" + importType.name();
+        }
 
         return appURL.setFragment(urlFragment).getLocalURIString();
     }

api/src/org/labkey/api/query/AbstractQueryImportAction.java

@@ -582,7 +582,7 @@ else if (!dataFileDir.exists())
                     multipartfile.transferTo(dataFile.toNioPathForWrite());
                     if (_useAsync)
                     {
-                        if (!isBackgroundImportSupported())
+                        if (!isBackgroundImportSupported(dataFile.getName()))
                             throw new RuntimeException("Importing in background currently is not supported for this table");
 
                         ViewBackgroundInfo info = new ViewBackgroundInfo(getContainer(), getUser(), new ActionURL());
@@ -757,7 +757,7 @@ protected String getQueryImportJobNotificationProviderName()
         return null;
     }
 
-    protected boolean isBackgroundImportSupported()
+    protected boolean isBackgroundImportSupported(@NotNull String fileName)
     {
         return false;
     }

api/src/org/labkey/api/query/AbstractQueryUpdateService.java

@@ -399,6 +399,11 @@ protected int _importRowsUsingDIB(User user, Container container, DataIteratorBu
         if (extraScriptContext != null)
         {
             context.setDataSource((String) extraScriptContext.get(DataIteratorUtil.DATA_SOURCE));
+            if (extraScriptContext.containsKey(AbstractQueryImportAction.Params.useTransactionAuditCache.name()))
+            {
+                boolean useTransactionAuditCache = Boolean.TRUE.equals(extraScriptContext.get(AbstractQueryImportAction.Params.useTransactionAuditCache.name()));
+                context.setUseTransactionAuditCache(useTransactionAuditCache);
+            }
         }
 
         preImportDIBValidation(in, null);

api/src/org/labkey/api/query/QueryImportPipelineJob.java

@@ -36,6 +36,8 @@ public class QueryImportPipelineJob extends PipelineJob
 
     private long _transactionAuditId;
 
+    private Map<String, Object> _additionalJobResponseInfo;
+
     protected QueryImportPipelineJob()
     {}
 
@@ -279,7 +281,7 @@ public URLHelper getStatusHref()
             }
 
             url =  target.getGridURL(getContainer());
-            if (_transactionAuditId > 0)
+            if (url != null && _transactionAuditId > 0)
                 url.addParameter("transactionAuditId", String.valueOf(_transactionAuditId));
 
             return url;
@@ -343,9 +345,6 @@ public void run()
             if (auditEvent != null)
                 _transactionAuditId = auditEvent.getRowId();
 
-            setStatus(TaskStatus.complete);
-            getLogger().info("Done importing {}. {} row(s) imported.", getDescription(), importedCount);
-
             if (notificationProvider != null)
             {
                 Map<String, Object> results = new HashMap<>();
@@ -359,8 +358,15 @@ public void run()
 
                 if (!diContext.getResponseInfo().isEmpty())
                     results.putAll(diContext.getResponseInfo());
-                notificationProvider.onJobSuccess(this, results);
+
+                _additionalJobResponseInfo = results;
             }
+
+            setStatus(TaskStatus.complete);
+            getLogger().info("Done importing {}. {} row(s) imported.", getDescription(), importedCount);
+
+            if (notificationProvider != null)
+                notificationProvider.onJobSuccess(this, getAdditionalJobResponseInfo());
         }
         catch (QueryImportJobCancelledException e)
         {
@@ -415,4 +421,9 @@ public long getTransactionAuditId()
         return _transactionAuditId;
     }
 
+    public Map<String, Object> getAdditionalJobResponseInfo()
+    {
+        return _additionalJobResponseInfo;
+    }
+
 }

audit/src/org/labkey/audit/AuditController.java

@@ -69,7 +69,6 @@
 import java.util.Collections;
 import java.util.Date;
 import java.util.GregorianCalendar;
-import java.util.List;
 import java.util.Map;
 
 import static org.labkey.api.data.ContainerManager.REQUIRE_USER_COMMENTS_PROPERTY_NAME;
@@ -382,17 +381,18 @@ public void validateForm(AuditTransactionForm form, Errors errors)
         @Override
         public Object execute(AuditTransactionForm form, BindException errors)
         {
-            List<Long> rowIds;
+            AuditLogImpl.TransactionRowIds results;
             User elevatedUser = ElevatedUser.ensureCanSeeAuditLogRole(getContainer(), getUser());
             ContainerFilter cf = ContainerFilter.getContainerFilterByName(form.getContainerFilter(), getContainer(), elevatedUser);
             if (form.isSampleType())
-                rowIds = AuditLogImpl.get().getTransactionSampleIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf);
+                results = AuditLogImpl.get().getTransactionSampleIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf);
             else
-                rowIds = AuditLogImpl.get().getTransactionSourceIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf);
+                results = AuditLogImpl.get().getTransactionSourceIds(form.getTransactionAuditId(), elevatedUser, getContainer(), cf);
 
             ApiSimpleResponse response = new ApiSimpleResponse();
             response.put("success", true);
-            response.put("rowIds", rowIds);
+            response.put("rowIds", results.rowIds());
+            response.put("dataTypeRowCounts", results.dataTypeRowCounts());
 
             return response;
         }