From 20254391dc8b56e60cf122ad8a5004712ee38f50 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 14 Nov 2024 11:12:49 +0000 Subject: [PATCH] Add structure encoder to transform structure features to match sequence dimensions --- models/analysis/multimodal_integrator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/analysis/multimodal_integrator.py b/models/analysis/multimodal_integrator.py index 465a0f6..fde949e 100644 --- a/models/analysis/multimodal_integrator.py +++ b/models/analysis/multimodal_integrator.py @@ -141,6 +141,14 @@ def __init__(self, hidden_size: int): nn.Dropout(0.1) ) + # Transform structure features to match sequence dimensions + self.structure_encoder = nn.Sequential( + nn.Linear(3, 768), # Transform structure features to match dimensions + nn.LayerNorm(768), # Normalize features + nn.ReLU(), + nn.Dropout(0.1) + ) + self.integration_network = nn.Sequential( nn.Linear(768 * 3, 1536), # Concatenated features from all three modalities nn.LayerNorm(1536), # Normalize combined features @@ -171,6 +179,10 @@ def forward( function_features = self.function_encoder(function_results['go_terms']) print(f"Transformed function features shape: {function_features.shape}") + # Transform structure features to match sequence dimensions + structure_features = self.structure_encoder(structure_features) + print(f"Transformed structure features shape: {structure_features.shape}") + # Ensure all features have the same dimensions before combining batch_size = sequence_features.size(0) seq_len = sequence_features.size(1)