Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: SpectralGPT for multi-temporal change detection #34

Merged
merged 8 commits into from
Sep 18, 2024
Merged

WIP: SpectralGPT for multi-temporal change detection #34

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8d172f8
Sets SpectralGPT encoder to monotemporal if UperNetCD is used. Otherw…
SebastianGer Sep 12, 2024
110f46c
Moves spectralgpt compatibility ensurance to config-level preprocessi…
SebastianGer Sep 17, 2024
5207a2f
Reverts most of the changes made to ensure compatibility between Spec…
SebastianGer Sep 18, 2024
4d0e9ba
MTUpernet semantic segmentation, treat Spectralgpt as a single tempor…
yurujaja Sep 18, 2024
309ec8c
Set num_frames=1 for SpectralGPT, instead of potentially using multit…
SebastianGer Sep 18, 2024
45b3fe3
Merge branch 'spectralgpt-multitemporal' of https://github.com/yuruja…
SebastianGer Sep 18, 2024
ba4357d
Merge branch 'main' into spectralgpt-multitemporal
SebastianGer Sep 18, 2024
73beb00
remove unnecessary repeat
yurujaja Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion foundation_models/spectralgpt_encoder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ def __init__(self,

self.model_name = "SpectralGPT"
self.output_layers = cfg['output_layers']
self.num_frames = cfg['multi_temporal'] if cfg['multi_temporal'] else 1
self.num_frames = 1

self.patch_embed = PatchEmbed(
img_size, patch_size, self.num_frames, embed_dim, in_chans, t_patch_size)
Expand Down
41 changes: 26 additions & 15 deletions segmentors/upernet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -202,25 +202,31 @@ def __init__(self, args, cfg, encoder, pool_scales=(1, 2, 3, 6)):
def forward(self, img, output_shape=None):
"""Forward function for change detection."""

if self.encoder.model_name not in ["Prithvi", "satlas_pretrain", "SpectralGPT"]:
if self.encoder.model_name in ["Prithvi", "satlas_pretrain"]:
if not self.finetune:
with torch.no_grad():
feats = self.encoder(img)
else:
feats = self.encoder(img)
else:
feats = []
for i in range(self.multi_temporal):
if not self.finetune:
with torch.no_grad():
feats.append(self.encoder({k: v[:,:,i,:,:] for k, v in img.items()}))
if self.encoder.model_name in ["SpectralGPT"]:
feats.append(self.encoder({k: v[:,:,[i],:,:] for k, v in img.items()}))
else:
feats.append(self.encoder({k: v[:,:,i,:,:] for k, v in img.items()}))
else:
feats.append(self.encoder({k: v[:,:,i,:,:] for k, v in img.items()}))
if self.encoder.model_name in ["SpectralGPT"]:
feats.append(self.encoder({k: v[:,:,[i],:,:] for k, v in img.items()}))
else:
feats.append(self.encoder({k: v[:,:,i,:,:] for k, v in img.items()}))

feats = [list(i) for i in zip(*feats)]
feats = [torch.stack(feat_layers, dim = 2) for feat_layers in feats]
else:
if not self.finetune:
with torch.no_grad():
feats = self.encoder(img)
else:
feats = self.encoder(img)

if self.encoder.model_name not in ["satlas_pretrain", "SpectralGPT"]:
feats = [torch.stack(feat_layers, dim = 2) for feat_layers in feats]

if self.encoder.model_name not in ["satlas_pretrain"]:
for i in range(len(feats)):
if self.multi_temporal_strategy == "ltae":
feats[i] = self.tmap(feats[i])
Expand Down Expand Up @@ -256,8 +262,13 @@ def forward(self, img, output_shape=None):
"""Forward function for change detection."""

if self.encoder.model_name != "Prithvi":
img1 = {k: v[:,:,0,:,:] for k, v in img.items()}
img2 = {k: v[:,:,1,:,:] for k, v in img.items()}
if self.encoder.model_name == "SpectralGPT":
# Retains the temporal dimension
img1 = {k: v[:,:,[0],:,:] for k, v in img.items()}
img2 = {k: v[:,:,[1],:,:] for k, v in img.items()}
else:
img1 = {k: v[:,:,0,:,:] for k, v in img.items()}
img2 = {k: v[:,:,1,:,:] for k, v in img.items()}

if not self.finetune:
with torch.no_grad():
Expand Down Expand Up @@ -413,4 +424,4 @@ def forward(self, inputs):
]
for i in range(len(inputs)):
outputs.append(ops[i](inputs[i]))
return tuple(outputs)
return tuple(outputs)