num-codex · hhund · Mar 25, 2024 · Oct 11, 2023 · Oct 11, 2023 · Mar 16, 2024
diff --git a/codex-process-data-transfer/pom.xml b/codex-process-data-transfer/pom.xml
@@ -8,7 +8,7 @@
 	<parent>
 		<groupId>de.netzwerk-universitaetsmedizin.codex</groupId>
 		<artifactId>codex-processes-ap1</artifactId>
-		<version>1.0.0.0-SNAPSHOT</version>
+		<version>1.1.0.0-SNAPSHOT</version>
 	</parent>
 
 	<properties>

diff --git a/...de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java b/...de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/ConstantsDataTransfer.java
@@ -91,6 +91,7 @@ public interface ConstantsDataTransfer
 	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_BAD_PATIENT_REFERENCE = "bad-patient-reference";
 	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_FTTP_NOT_REACHABLE = "fttp-not-reachable";
 	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_NO_DIC_PSEUDONYM_FOR_BLOOMFILTER = "no-dic-pseudonym-for-bloomfilter";
+	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_NO_DIC_PSEUDONYM_FOR_LOCAL_PSEUDONYM = "no-dic-pseudonym-for-local-pseudonym";
 	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_VALIDATION_FAILED = "validation-failed";
 	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_ECRYPTION_OF_DATA_FOR_CRR_FAILED = "ecryption-of-data-for-crr-failed";
 	String CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_UNABLE_TO_STORE_ECRYPTED_DATA = "unable-to-store-ecrypted-data";

diff --git a/...iversitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java b/...iversitaetsmedizin/codex/processes/data_transfer/DataTransferProcessPluginDefinition.java
@@ -17,8 +17,8 @@
 
 public class DataTransferProcessPluginDefinition implements ProcessPluginDefinition
 {
-	public static final String VERSION = "1.0.0.0";
-	public static final LocalDate DATE = LocalDate.of(2023, 9, 25);
+	public static final String VERSION = "1.1.0.0";
+	public static final LocalDate DATE = LocalDate.of(2024, 3, 18);
 
 	@Override
 	public String getName()

diff --git a/...ava/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/FttpClient.java b/...ava/de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/FttpClient.java
@@ -18,5 +18,7 @@ public interface FttpClient
 	 */
 	Optional<String> getDicPseudonym(String bloomFilter);
 
+	Optional<String> getDicPseudonymForLocalPseudonym(String localPseudonym);
+
 	void testConnection();
 }
diff --git a/...netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/FttpClientFactory.java b/...netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/FttpClientFactory.java
@@ -62,6 +62,18 @@ public Optional<String> getDicPseudonym(String bloomFilter)
 			return pseudonym;
 		}
 
+		@Override
+		public Optional<String> getDicPseudonymForLocalPseudonym(String localPseudonym)
+		{
+			Optional<String> pseudonym = sha256(localPseudonym).map(p -> "dic_test/" + p);
+
+			logger.warn(
+					"Returning simulated DIC pseudonym '{}' for local pseudonym '{}', fTTP connection not configured.",
+					pseudonym.orElseThrow(), localPseudonym);
+
+			return pseudonym;
+		}
+
 		private Optional<String> sha256(String original)
 		{
 			try
@@ -142,7 +154,7 @@ public void testConnection()
 		{
 			logger.info(
 					"Testing connection to fTTP with {trustStorePath: {}, certificatePath: {}, privateKeyPath: {}, privateKeyPassword: {},"
-							+ " basicAuthUsername {}, basicAuthPassword {}, serverBase: {}, apiKey: {}, study: {}, target: {}, proxyUrl {}, proxyUsername, proxyPassword {}}",
+							+ " basicAuthUsername: {}, basicAuthPassword: {}, serverBase: {}, apiKey: {}, study: {}, target: {}, proxyUrl: {}, proxyUsername: {}, proxyPassword: {}}",
 					trustStorePath, certificatePath, privateKeyPath, privateKeyPassword != null ? "***" : "null",
 					fttpBasicAuthUsername, fttpBasicAuthPassword != null ? "***" : "null", fttpServerBase,
 					fttpApiKey != null ? "***" : "null", fttpStudy, fttpTarget, proxyUrl, proxyUsername,

diff --git a/...de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/FttpClientImpl.java b/...de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/client/FttpClientImpl.java
@@ -154,6 +154,18 @@ protected Parameters createParametersForPsnWorkflow(String dicSourceAndPseudonym
 		return p;
 	}
 
+	protected Parameters createParametersForPsnWorkflowLocalPseudonym(String localPseudonym)
+	{
+		Parameters p = new Parameters();
+		p.addParameter("study", fttpStudy);
+		p.addParameter("original", localPseudonym);
+		p.addParameter("source", "local");
+		p.addParameter("target", fttpTarget);
+		p.addParameter("apikey", fttpApiKey);
+
+		return p;
+	}
+
 	@Override
 	public Optional<String> getDicPseudonym(String bloomFilter)
 	{
@@ -178,6 +190,30 @@ public Optional<String> getDicPseudonym(String bloomFilter)
 		}
 	}
 
+	@Override
+	public Optional<String> getDicPseudonymForLocalPseudonym(String localPseudonym)
+	{
+		Objects.requireNonNull(localPseudonym, "localPseudonym");
+
+		logger.info("Requesting DIC Pseudonym for local Pseudonym {} ...", localPseudonym);
+
+		try
+		{
+			IGenericClient client = createGenericClient();
+
+			Parameters parameters = client.operation().onServer().named("requestPsnWorkflow")
+					.withParameters(createParametersForPsnWorkflowLocalPseudonym(localPseudonym))
+					.accept(Constants.CT_FHIR_XML_NEW).encoded(EncodingEnum.XML).execute();
+
+			return getPseudonym(parameters).map(p -> fttpTarget + "/" + p);
+		}
+		catch (Exception e)
+		{
+			logger.error("Error while retrieving DIC pseudonym: {} - {}", e.getClass().getName(), e.getMessage());
+			throw new BpmnError(CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_FTTP_NOT_REACHABLE, e.getMessage());
+		}
+	}
+
 	protected Parameters createParametersForBfWorkflow(String bloomFilter)
 	{
 		Parameters p = new Parameters();

diff --git a/...rk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java b/...rk_universitaetsmedizin/codex/processes/data_transfer/client/fhir/AbstractFhirClient.java
@@ -653,6 +653,8 @@ public void updatePatient(Patient patient)
 		Objects.requireNonNull(patient, "patient");
 
 		String id = patient.getIdElement().toVersionless().getValue();
+		// set the patient id to versionless id to workaround a `If-Match`-header bug in hapi fhir client
+		patient.setId(id);
 		logger.info("Updating patient {}", id);
 
 		try

diff --git a/...aetsmedizin/codex/processes/data_transfer/service/receive/StoreValidationErrorForDts.java b/...aetsmedizin/codex/processes/data_transfer/service/receive/StoreValidationErrorForDts.java
@@ -15,7 +15,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import ca.uhn.fhir.context.FhirContext;
+import de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.logging.DataLogger;
 import dev.dsf.bpe.v1.ProcessPluginApi;
 import dev.dsf.bpe.v1.activity.AbstractServiceDelegate;
 import dev.dsf.bpe.v1.constants.NamingSystems;
@@ -26,12 +26,14 @@ public class StoreValidationErrorForDts extends AbstractServiceDelegate
 	private static final Logger logger = LoggerFactory.getLogger(StoreValidationErrorForDts.class);
 
 	private final String dtsIdentifierValue;
+	private final DataLogger dataLogger;
 
-	public StoreValidationErrorForDts(ProcessPluginApi api, String dtsIdentifierValue)
+	public StoreValidationErrorForDts(ProcessPluginApi api, String dtsIdentifierValue, DataLogger dataLogger)
 	{
 		super(api);
 
 		this.dtsIdentifierValue = dtsIdentifierValue;
+		this.dataLogger = dataLogger;
 	}
 
 	@Override
@@ -40,6 +42,7 @@ public void afterPropertiesSet() throws Exception
 		super.afterPropertiesSet();
 
 		Objects.requireNonNull(dtsIdentifierValue, "dtsIdentifierValue");
+		Objects.requireNonNull(dataLogger, "dataLogger");
 	}
 
 	@Override
@@ -72,7 +75,7 @@ private IdType createBinaryResource(Binary binary)
 		}
 		catch (Exception e)
 		{
-			logger.debug("Binary to create {}", FhirContext.forR4().newJsonParser().encodeResourceToString(binary));
+			dataLogger.logData("Binary to create", binary);
 			logger.warn("Error while creating Binary resource: " + e.getMessage(), e);
 
 			throw new BpmnError(

diff --git a/...de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/send/ReadData.java b/...de/netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/send/ReadData.java
@@ -25,6 +25,7 @@
 
 import org.camunda.bpm.engine.delegate.BpmnError;
 import org.camunda.bpm.engine.delegate.DelegateExecution;
+import org.hl7.fhir.instance.model.api.IBaseResource;
 import org.hl7.fhir.r4.model.Bundle;
 import org.hl7.fhir.r4.model.Bundle.BundleEntryComponent;
 import org.hl7.fhir.r4.model.Bundle.BundleType;
@@ -69,6 +70,7 @@
 import org.hl7.fhir.r4.model.Procedure.ProcedurePerformerComponent;
 import org.hl7.fhir.r4.model.Reference;
 import org.hl7.fhir.r4.model.Resource;
+import org.hl7.fhir.r4.model.Specimen;
 import org.hl7.fhir.r4.model.Task;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -176,6 +178,7 @@ protected Bundle toBundle(String pseudonym, Stream<DomainResource> resourcesStre
 				.collect(Collectors.toMap(r -> r.getIdElement().toUnqualifiedVersionless().getValue(),
 						r -> "urn:uuid:" + UUID.randomUUID().toString()));
 
+
 		List<DomainResource> resourcesWithTemporaryReferences = fixReferences(resources, resourcesById, uuidsById);
 		List<BundleEntryComponent> entries = resourcesWithTemporaryReferences.stream().map(r ->
 		{
@@ -240,12 +243,20 @@ else if (resource instanceof Observation o)
 					Observation::setEncounter);
 			fixReferences(o, uuidsById, "Observation.hasMember", Observation::hasHasMember, Observation::getHasMember,
 					Observation::setHasMember);
+			fixReference(o, uuidsById, "Observation.specimen", Observation::hasSpecimen, Observation::getSpecimen,
+					Observation::setSpecimen);
 		}
 		else if (resource instanceof Procedure p)
 		{
 			fixReference(p, uuidsById, "Procedure.encounter", Procedure::hasEncounter, Procedure::getEncounter,
 					Procedure::setEncounter);
 		}
+		else if (resource instanceof Specimen s)
+		{
+			fixReferences(s, uuidsById, "Specimen.parent", Specimen::hasParent, Specimen::getParent,
+					Specimen::setParent);
+		}
+
 
 		return resource;
 	}
@@ -269,6 +280,18 @@ else if (oldReference.hasReference() && oldReference.getReference() == null && o
 			{
 				logger.debug("Not removing empty reference at {} with data-absent-reason extension", path);
 			}
+			else if (!oldReference.getResource().isEmpty())
+			{
+				String internalId = "#" + UUID.randomUUID();
+				Reference fixedReference = new Reference(internalId);
+				IBaseResource oldContainedResource = clean((DomainResource) oldReference.getResource());
+				oldContainedResource.setId(internalId);
+				fixedReference.setResource(oldContainedResource);
+				setReference.apply(resource, fixedReference);
+				logger.debug(
+						"Replacing reference to contained resource at {} from resource {} with bundle temporary id in transport bundle",
+						path, getAbsoluteId(resource).getValue());
+			}
 			else
 			{
 				logger.warn("Removing reference at {} from resource {} in transport bundle", path,
@@ -307,6 +330,18 @@ else if (oldReference.hasReference() && oldReference.getReference() == null
 					logger.debug("Not removing empty reference at {}[{}] with data-absent-reason extension", path, i);
 					fixedReferences.add(oldReference);
 				}
+				else if (!oldReference.getResource().isEmpty())
+				{
+					String internalId = "#" + UUID.randomUUID();
+					Reference fixedReference = new Reference(internalId);
+					IBaseResource oldContainedResource = clean((DomainResource) oldReference.getResource());
+					oldContainedResource.setId(internalId);
+					fixedReference.setResource(oldContainedResource);
+					fixedReferences.add(fixedReference);
+					logger.debug(
+							"Replacing reference to contained resource at {}[{}] from resource {} with bundle temporary id in transport bundle",
+							path, i, getAbsoluteId(resource).getValue());
+				}
 				else
 				{
 					logger.warn("Removing reference at {}[{}] from resource {} in transport bundle", path, i,
@@ -342,9 +377,10 @@ private <R extends DomainResource, C> R fixReferenceFromComponents(R resource, M
 							i, getAbsoluteId(resource).getValue());
 					setReference.apply(component, new Reference(uuidsById.get(oldReference.getReference())));
 				}
-				else if (oldReference.hasReference() && oldReference.getReference() == null
+				else if ((oldReference.hasReference() && oldReference.getReference() == null
 						&& oldReference.getReferenceElement_()
 								.hasExtension("http://hl7.org/fhir/StructureDefinition/data-absent-reason"))
+						|| oldReference.hasExtension("http://hl7.org/fhir/StructureDefinition/data-absent-reason"))
 				{
 					logger.debug("Not removing empty reference at " + path + " with data-absent-reason extension", i);
 				}
@@ -645,7 +681,6 @@ else if (resource instanceof Observation o)
 			cleanUnsupportedReferences(o, "Observation.focus", Observation::hasFocus, Observation::setFocus);
 			cleanUnsupportedReferences(o, "Observation.performer", Observation::hasPerformer,
 					Observation::setPerformer);
-			cleanUnsupportedReference(o, "Observation.specimen", Observation::hasSpecimen, Observation::setSpecimen);
 			cleanUnsupportedReference(o, "Observation.device", Observation::hasDevice, Observation::setDevice);
 			cleanUnsupportedReferences(o, "Observation.derivedFrom", Observation::hasDerivedFrom,
 					Observation::setDerivedFrom);
@@ -674,6 +709,20 @@ else if (resource instanceof Procedure p)
 			cleanUnsupportedReferences(p, "Procedure.usedReference", Procedure::hasUsedReference,
 					Procedure::setUsedReference);
 		}
+		else if (resource instanceof Specimen s)
+		{
+			cleanUnsupportedReferences(s, "Specimen.request", Specimen::hasRequest, Specimen::setRequest);
+			cleanUnsupportedReferenceFromComponent(s, "Specimen.collection.collector", Specimen::hasCollection,
+					Specimen::getCollection, Specimen.SpecimenCollectionComponent::hasCollector,
+					Specimen.SpecimenCollectionComponent::setCollector);
+			cleanUnsupportedReferencesFromComponents(s, "Specimen.processing[{}].additive", Specimen::hasProcessing,
+					Specimen::getProcessing, Specimen.SpecimenProcessingComponent::hasAdditive,
+					Specimen.SpecimenProcessingComponent::setAdditive);
+			cleanUnsupportedReferenceFromComponents(s, "Specimen.container[{}].additiveReference",
+					Specimen::hasContainer, Specimen::getContainer,
+					Specimen.SpecimenContainerComponent::hasAdditiveReference,
+					Specimen.SpecimenContainerComponent::setAdditive);
+		}
 		else
 			throw new RuntimeException("Resource of type " + resource.getResourceType().name() + " not supported");
 	}
@@ -741,10 +790,21 @@ else if (resource instanceof Procedure p)
 				p.setIdentifier(Collections.emptyList());
 				p.setSubject(patientRef);
 			}
+			else if (resource instanceof Specimen s)
+			{
+				s.setIdentifier(Collections.emptyList());
+				s.setAccessionIdentifier(null);
+				s.setSubject(patientRef);
+			}
 			else
 				throw new RuntimeException("Resource of type " + resource.getResourceType().name() + " not supported");
 		}
 
+		if (resource instanceof DomainResource d)
+			d.getContained().forEach(r -> setSubjectOrIdentifier(r, pseudonym));
+		else
+			throw new RuntimeException("Resource of type " + resource.getResourceType().name() + " not supported");
+
 		return resource;
 	}
 

diff --git a/.../netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/send/ResolvePsn.java b/.../netzwerk_universitaetsmedizin/codex/processes/data_transfer/service/send/ResolvePsn.java
@@ -2,11 +2,13 @@
 
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.BPMN_EXECUTION_VARIABLE_PATIENT_REFERENCE;
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_NO_DIC_PSEUDONYM_FOR_BLOOMFILTER;
+import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_NO_DIC_PSEUDONYM_FOR_LOCAL_PSEUDONYM;
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_PATIENT_NOT_FOUND;
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.IDENTIFIER_NUM_CODEX_DIC_PSEUDONYM_TYPE_CODE;
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.IDENTIFIER_NUM_CODEX_DIC_PSEUDONYM_TYPE_SYSTEM;
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.NAMING_SYSTEM_NUM_CODEX_BLOOM_FILTER;
 import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.NAMING_SYSTEM_NUM_CODEX_DIC_PSEUDONYM;
+import static de.netzwerk_universitaetsmedizin.codex.processes.data_transfer.ConstantsDataTransfer.RFC_4122_SYSTEM;
 
 import java.util.Objects;
 import java.util.Optional;
@@ -94,8 +96,33 @@ private Optional<String> getPseudonym(Patient patient)
 
 	private String resolvePseudonymAndUpdatePatient(Patient patient)
 	{
-		String bloomFilter = getBloomFilter(patient);
-		String pseudonym = resolveBloomFilter(bloomFilter);
+		String pseudonym;
+		// first try to find a bloom filter
+		Optional<String> bloomFilter = getBloomFilter(patient);
+		if (bloomFilter.isPresent())
+		{
+			pseudonym = resolveBloomFilter(bloomFilter.get());
+		}
+		else
+		{
+			// otherwise try to find a local pseudonym
+			// --> no record linkage
+			logger.info(
+					"No bloom filter present for patient {}. Try to use the local pseudonym for data transfer without record linkage",
+					patient.getIdElement().getValue());
+			Optional<String> localPseudonym = getLocalPseudonym(patient);
+			if (localPseudonym.isPresent())
+			{
+				pseudonym = resolveLocalPseudonym(localPseudonym.get());
+			}
+			else
+			{
+				logger.info("No local pseudonym present for patient {}. Aborted", patient.getIdElement().getValue());
+				throw new RuntimeException("Could not find pseudonym");
+			}
+
+
+		}
 
 		patient.getIdentifier().removeIf(i -> NAMING_SYSTEM_NUM_CODEX_BLOOM_FILTER.equals(i.getSystem()));
 		patient.addIdentifier().setSystem(NAMING_SYSTEM_NUM_CODEX_DIC_PSEUDONYM).setValue(pseudonym).getType()
@@ -107,12 +134,18 @@ private String resolvePseudonymAndUpdatePatient(Patient patient)
 		return pseudonym;
 	}
 
-	private String getBloomFilter(Patient patient)
+	private Optional<String> getBloomFilter(Patient patient)
 	{
 		return patient.getIdentifier().stream().filter(Identifier::hasSystem)
 				.filter(i -> NAMING_SYSTEM_NUM_CODEX_BLOOM_FILTER.equals(i.getSystem())).filter(Identifier::hasValue)
-				.findFirst().map(Identifier::getValue).orElseThrow(() -> new RuntimeException(
-						"No bloom filter present in patient " + patient.getIdElement().getValue()));
+				.findFirst().map(Identifier::getValue);
+	}
+
+	private Optional<String> getLocalPseudonym(Patient patient)
+	{
+		return patient.getIdentifier().stream().filter(Identifier::hasSystem)
+				.filter(i -> RFC_4122_SYSTEM.equals(i.getSystem())).filter(Identifier::hasValue).findFirst()
+				.map(Identifier::getValue);
 	}
 
 	private String resolveBloomFilter(String bloomFilter)
@@ -123,6 +156,15 @@ private String resolveBloomFilter(String bloomFilter)
 						"Unable to get DIC pseudonym for given BloomFilter"));
 	}
 
+	private String resolveLocalPseudonym(String localPseudonym)
+	{
+		return fttpClientFactory.getFttpClient().getDicPseudonymForLocalPseudonym(localPseudonym)
+				.orElseThrow(() -> new BpmnError(
+						CODESYSTEM_NUM_CODEX_DATA_TRANSFER_ERROR_VALUE_NO_DIC_PSEUDONYM_FOR_LOCAL_PSEUDONYM,
+						"Unable to get DIC pseudonym for given localPseudonym"));
+	}
+
+
 	private void updatePatient(Patient patient)
 	{
 		dataStoreClientFactory.getDataStoreClient().getFhirClient().updatePatient(patient);