Initial media depth project backup
This commit is contained in:
Codex
55
Depth-Anything-V1-main/semseg/README.md
Normal file
55
Depth-Anything-V1-main/semseg/README.md
Normal file
@@ -0,0 +1,55 @@
|
||||
# Depth Anything for Semantic Segmentation
|
||||
|
||||
We use our Depth Anything pre-trained ViT-L encoder to fine-tune downstream semantic segmentation models.
|
||||
|
||||
|
||||
## Performance
|
||||
|
||||
### Cityscapes
|
||||
|
||||
Note that our results are obtained *without* Mapillary pre-training.
|
||||
|
||||
| Method | Encoder | mIoU (s.s.) | m.s. |
|
||||
|:-:|:-:|:-:|:-:|
|
||||
| SegFormer | MiT-B5 | 82.4 | 84.0 |
|
||||
| Mask2Former | Swin-L | 83.3 | 84.3 |
|
||||
| OneFormer | Swin-L | 83.0 | 84.4 |
|
||||
| OneFormer | ConNeXt-XL | 83.6 | 84.6 |
|
||||
| DDP | ConNeXt-L | 83.2 | 83.9 |
|
||||
| **Ours** | ViT-L | **84.8** | **86.2** |
|
||||
|
||||
|
||||
### ADE20K
|
||||
|
||||
| Method | Encoder | mIoU |
|
||||
|:-:|:-:|:-:|
|
||||
| SegFormer | MiT-B5 | 51.0 |
|
||||
| Mask2Former | Swin-L | 56.4 |
|
||||
| UperNet | BEiT-L | 56.3 |
|
||||
| ViT-Adapter | BEiT-L | 58.3 |
|
||||
| OneFormer | Swin-L | 57.4 |
|
||||
| OneFormer | ConNeXt-XL | 57.4 |
|
||||
| **Ours** | ViT-L | **59.4** |
|
||||
|
||||
|
||||
## Pre-trained models
|
||||
|
||||
- [Cityscapes-ViT-L-mIoU-86.4](https://huggingface.co/spaces/LiheYoung/Depth-Anything/blob/main/checkpoints_semseg/cityscapes_vitl_mIoU_86.4.pth)
|
||||
- [ADE20K-ViT-L-mIoU-59.4](https://huggingface.co/spaces/LiheYoung/Depth-Anything/blob/main/checkpoints_semseg/ade20k_vitl_mIoU_59.4.pth)
|
||||
|
||||
|
||||
## Installation
|
||||
|
||||
Please refer to [MMSegmentation](https://github.com/open-mmlab/mmsegmentation/blob/main/docs/en/get_started.md#installation) for instructions. *Do not forget to install ``mmdet`` to support ``Mask2Former``:*
|
||||
```bash
|
||||
pip install "mmdet>=3.0.0rc4"
|
||||
```
|
||||
|
||||
After installation:
|
||||
- move our [config/depth_anything](./config/depth_anything/) to mmseg's [config](https://github.com/open-mmlab/mmsegmentation/tree/main/configs)
|
||||
- move our [dinov2.py](./dinov2.py) to mmseg's [backbones](https://github.com/open-mmlab/mmsegmentation/tree/main/mmseg/models/backbones)
|
||||
- add DINOv2 in mmseg's [models/backbones/\_\_init\_\_.py](https://github.com/open-mmlab/mmsegmentation/blob/main/mmseg/models/backbones/__init__.py)
|
||||
- download our provided [torchhub](https://github.com/LiheYoung/Depth-Anything/tree/main/torchhub) directory and put it at the root of your working directory
|
||||
- download the [Depth Anything pre-trained model](https://huggingface.co/spaces/LiheYoung/Depth-Anything/blob/main/checkpoints/depth_anything_vitl14.pth) (to initialize the encoder) and 2) put it under the ``checkpoints`` folder.
|
||||
|
||||
For training or inference with our pre-trained models, please refer to MMSegmentation [instructions](https://github.com/open-mmlab/mmsegmentation/blob/main/docs/en/user_guides/4_train_test.md).
|
||||
@@ -0,0 +1,230 @@
|
||||
_base_ = [
|
||||
'../_base_/default_runtime.py', '../_base_/datasets/ade20k_640x640.py'
|
||||
]
|
||||
|
||||
crop_size = (896, 896)
|
||||
data_preprocessor = dict(
|
||||
type='SegDataPreProcessor',
|
||||
mean=[123.675, 116.28, 103.53],
|
||||
std=[58.395, 57.12, 57.375],
|
||||
bgr_to_rgb=True,
|
||||
pad_val=0,
|
||||
seg_pad_val=255,
|
||||
size=crop_size)
|
||||
num_classes = 150
|
||||
|
||||
model = dict(
|
||||
type='EncoderDecoder',
|
||||
data_preprocessor=data_preprocessor,
|
||||
backbone=dict(
|
||||
type='DINOv2',
|
||||
version='large',
|
||||
freeze=False,
|
||||
load_from='./checkpoints/depth_anything_vitl14.pth'),
|
||||
neck=dict(type='Feature2Pyramid', embed_dim=1024, rescales=[4, 2, 1, 0.5]),
|
||||
decode_head=dict(
|
||||
type='Mask2FormerHead',
|
||||
in_channels=[1024, 1024, 1024, 1024],
|
||||
# strides=[4, 8, 16, 32],
|
||||
feat_channels=1024,
|
||||
out_channels=1024,
|
||||
num_classes=num_classes,
|
||||
num_queries=200,
|
||||
num_transformer_feat_level=3,
|
||||
align_corners=False,
|
||||
pixel_decoder=dict(
|
||||
type='mmdet.MSDeformAttnPixelDecoder',
|
||||
num_outs=3,
|
||||
norm_cfg=dict(type='GN', num_groups=32),
|
||||
act_cfg=dict(type='ReLU'),
|
||||
encoder=dict( # DeformableDetrTransformerEncoder
|
||||
num_layers=6,
|
||||
layer_cfg=dict( # DeformableDetrTransformerEncoderLayer
|
||||
self_attn_cfg=dict( # MultiScaleDeformableAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
num_levels=3,
|
||||
num_points=4,
|
||||
im2col_step=64,
|
||||
dropout=0.0,
|
||||
batch_first=True,
|
||||
norm_cfg=None,
|
||||
init_cfg=None),
|
||||
ffn_cfg=dict(
|
||||
embed_dims=1024,
|
||||
feedforward_channels=4096,
|
||||
num_fcs=2,
|
||||
ffn_drop=0.0,
|
||||
act_cfg=dict(type='ReLU', inplace=True))),
|
||||
init_cfg=None),
|
||||
positional_encoding=dict( # SinePositionalEncoding
|
||||
num_feats=512, normalize=True),
|
||||
init_cfg=None),
|
||||
enforce_decoder_input_project=False,
|
||||
positional_encoding=dict( # SinePositionalEncoding
|
||||
num_feats=512, normalize=True),
|
||||
transformer_decoder=dict( # Mask2FormerTransformerDecoder
|
||||
return_intermediate=True,
|
||||
num_layers=9,
|
||||
layer_cfg=dict( # Mask2FormerTransformerDecoderLayer
|
||||
self_attn_cfg=dict( # MultiheadAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
attn_drop=0.0,
|
||||
proj_drop=0.0,
|
||||
dropout_layer=None,
|
||||
batch_first=True),
|
||||
cross_attn_cfg=dict( # MultiheadAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
attn_drop=0.0,
|
||||
proj_drop=0.0,
|
||||
dropout_layer=None,
|
||||
batch_first=True),
|
||||
ffn_cfg=dict(
|
||||
embed_dims=1024,
|
||||
feedforward_channels=4096,
|
||||
num_fcs=2,
|
||||
act_cfg=dict(type='ReLU', inplace=True),
|
||||
ffn_drop=0.0,
|
||||
dropout_layer=None,
|
||||
add_identity=True)),
|
||||
init_cfg=None),
|
||||
loss_cls=dict(
|
||||
type='mmdet.CrossEntropyLoss',
|
||||
use_sigmoid=False,
|
||||
loss_weight=2.0,
|
||||
reduction='mean',
|
||||
class_weight=[1.0] * num_classes + [0.1]),
|
||||
loss_mask=dict(
|
||||
type='mmdet.CrossEntropyLoss',
|
||||
use_sigmoid=True,
|
||||
reduction='mean',
|
||||
loss_weight=5.0),
|
||||
loss_dice=dict(
|
||||
type='mmdet.DiceLoss',
|
||||
use_sigmoid=True,
|
||||
activate=True,
|
||||
reduction='mean',
|
||||
naive_dice=True,
|
||||
eps=1.0,
|
||||
loss_weight=5.0),
|
||||
train_cfg=dict(
|
||||
num_points=12544,
|
||||
oversample_ratio=3.0,
|
||||
importance_sample_ratio=0.75,
|
||||
assigner=dict(
|
||||
type='mmdet.HungarianAssigner',
|
||||
match_costs=[
|
||||
dict(type='mmdet.ClassificationCost', weight=2.0),
|
||||
dict(
|
||||
type='mmdet.CrossEntropyLossCost',
|
||||
weight=5.0,
|
||||
use_sigmoid=True),
|
||||
dict(
|
||||
type='mmdet.DiceCost',
|
||||
weight=5.0,
|
||||
pred_act=True,
|
||||
eps=1.0)
|
||||
]),
|
||||
sampler=dict(type='mmdet.MaskPseudoSampler'))),
|
||||
train_cfg=dict(),
|
||||
test_cfg=dict(mode='slide', crop_size=crop_size, stride=(426, 426)))
|
||||
|
||||
# dataset config
|
||||
train_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='LoadAnnotations', reduce_zero_label=True),
|
||||
dict(
|
||||
type='RandomChoiceResize',
|
||||
scales=[int(x * 0.1 * 896) for x in range(5, 21)],
|
||||
resize_type='ResizeShortestEdge',
|
||||
max_size=3584),
|
||||
dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75),
|
||||
dict(type='RandomFlip', prob=0.5),
|
||||
dict(type='PhotoMetricDistortion'),
|
||||
dict(type='PackSegInputs')
|
||||
]
|
||||
test_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='Resize', scale=(3584, 896), keep_ratio=True),
|
||||
# add loading annotation after ``Resize`` because ground truth
|
||||
# does not need to do resize data transform
|
||||
dict(type='LoadAnnotations', reduce_zero_label=True),
|
||||
dict(type='PackSegInputs')
|
||||
]
|
||||
train_dataloader = dict(batch_size=1, dataset=dict(pipeline=train_pipeline))
|
||||
val_dataloader = dict(dataset=dict(pipeline=test_pipeline))
|
||||
test_dataloader = val_dataloader
|
||||
|
||||
# optim_wrapper = dict(
|
||||
# _delete_=True,
|
||||
# type='OptimWrapper',
|
||||
# optimizer=dict(
|
||||
# type='AdamW', lr=3e-5, betas=(0.9, 0.999), weight_decay=0.05),
|
||||
# constructor='LayerDecayOptimizerConstructor',
|
||||
# paramwise_cfg=dict(num_layers=12, layer_decay_rate=0.9))
|
||||
|
||||
# set all layers in backbone to lr_mult=0.1
|
||||
# set all norm layers, position_embeding,
|
||||
# query_embeding, level_embeding to decay_multi=0.0
|
||||
backbone_norm_multi = dict(lr_mult=0.1, decay_mult=0.0)
|
||||
backbone_embed_multi = dict(lr_mult=0.1, decay_mult=0.0)
|
||||
embed_multi = dict(lr_mult=1.0, decay_mult=0.0)
|
||||
custom_keys = {
|
||||
'backbone.dinov2': dict(lr_mult=0.1, decay_mult=1.0),
|
||||
'backbone.dinov2.norm': backbone_norm_multi,
|
||||
'pos_embed': backbone_embed_multi,
|
||||
'query_embed': embed_multi,
|
||||
'query_feat': embed_multi,
|
||||
'level_embed': embed_multi
|
||||
}
|
||||
custom_keys.update({
|
||||
f'backbone.dinov2.blocks.{block_id}.norm': backbone_norm_multi
|
||||
for block_id in range(24)
|
||||
})
|
||||
# optimizer
|
||||
optimizer = dict(
|
||||
type='AdamW', lr=0.00003, weight_decay=0.05, eps=1e-8, betas=(0.9, 0.999))
|
||||
optim_wrapper = dict(
|
||||
type='OptimWrapper',
|
||||
optimizer=optimizer,
|
||||
clip_grad=dict(max_norm=0.01, norm_type=2),
|
||||
paramwise_cfg=dict(custom_keys=custom_keys, norm_decay_mult=0.0))
|
||||
|
||||
find_unused_parameters=True
|
||||
|
||||
param_scheduler = [
|
||||
dict(
|
||||
type='LinearLR', start_factor=1e-6, by_epoch=False, begin=0, end=1500),
|
||||
dict(
|
||||
type='PolyLR',
|
||||
power=1.0,
|
||||
begin=1500,
|
||||
end=160000,
|
||||
eta_min=0.0,
|
||||
by_epoch=False,
|
||||
)
|
||||
]
|
||||
|
||||
# training schedule for 160k
|
||||
train_cfg = dict(
|
||||
type='IterBasedTrainLoop', max_iters=160000, val_interval=5000)
|
||||
val_cfg = dict(type='ValLoop')
|
||||
test_cfg = dict(type='TestLoop')
|
||||
default_hooks = dict(
|
||||
timer=dict(type='IterTimerHook'),
|
||||
logger=dict(type='LoggerHook', interval=50, log_metric_by_epoch=False),
|
||||
param_scheduler=dict(type='ParamSchedulerHook'),
|
||||
checkpoint=dict(
|
||||
type='CheckpointHook', by_epoch=False, interval=5000, save_best='mIoU', max_keep_ckpts=1),
|
||||
sampler_seed=dict(type='DistSamplerSeedHook'),
|
||||
visualization=dict(type='SegVisualizationHook'))
|
||||
|
||||
# Default setting for scaling LR automatically
|
||||
# - `enable` means enable scaling LR automatically
|
||||
# or not by default.
|
||||
# - `base_batch_size` = (8 GPUs) x (2 samples per GPU).
|
||||
auto_scale_lr = dict(enable=False, base_batch_size=16)
|
||||
|
||||
work_dir = './work_dirs/depth_anything_large_mask2former_16xb1_160k_ade20k_896x896'
|
||||
@@ -0,0 +1,222 @@
|
||||
_base_ = [
|
||||
'../_base_/default_runtime.py', '../_base_/datasets/cityscapes.py'
|
||||
]
|
||||
|
||||
crop_size = (896, 896)
|
||||
data_preprocessor = dict(
|
||||
type='SegDataPreProcessor',
|
||||
mean=[123.675, 116.28, 103.53],
|
||||
std=[58.395, 57.12, 57.375],
|
||||
bgr_to_rgb=True,
|
||||
pad_val=0,
|
||||
seg_pad_val=255,
|
||||
size=crop_size)
|
||||
num_classes = 19
|
||||
|
||||
model = dict(
|
||||
type='EncoderDecoder',
|
||||
data_preprocessor=data_preprocessor,
|
||||
backbone=dict(
|
||||
type='DINOv2',
|
||||
version='large',
|
||||
freeze=False,
|
||||
load_from='./checkpoints/depth_anything_vitl14.pth'),
|
||||
neck=dict(type='Feature2Pyramid', embed_dim=1024, rescales=[4, 2, 1, 0.5]),
|
||||
decode_head=dict(
|
||||
type='Mask2FormerHead',
|
||||
in_channels=[1024, 1024, 1024, 1024],
|
||||
# strides=[4, 8, 16, 32],
|
||||
feat_channels=1024,
|
||||
out_channels=1024,
|
||||
num_classes=num_classes,
|
||||
num_queries=200,
|
||||
num_transformer_feat_level=3,
|
||||
align_corners=False,
|
||||
pixel_decoder=dict(
|
||||
type='mmdet.MSDeformAttnPixelDecoder',
|
||||
num_outs=3,
|
||||
norm_cfg=dict(type='GN', num_groups=32),
|
||||
act_cfg=dict(type='ReLU'),
|
||||
encoder=dict( # DeformableDetrTransformerEncoder
|
||||
num_layers=6,
|
||||
layer_cfg=dict( # DeformableDetrTransformerEncoderLayer
|
||||
self_attn_cfg=dict( # MultiScaleDeformableAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
num_levels=3,
|
||||
num_points=4,
|
||||
im2col_step=64,
|
||||
dropout=0.0,
|
||||
batch_first=True,
|
||||
norm_cfg=None,
|
||||
init_cfg=None),
|
||||
ffn_cfg=dict(
|
||||
embed_dims=1024,
|
||||
feedforward_channels=4096,
|
||||
num_fcs=2,
|
||||
ffn_drop=0.0,
|
||||
act_cfg=dict(type='ReLU', inplace=True))),
|
||||
init_cfg=None),
|
||||
positional_encoding=dict( # SinePositionalEncoding
|
||||
num_feats=512, normalize=True),
|
||||
init_cfg=None),
|
||||
enforce_decoder_input_project=False,
|
||||
positional_encoding=dict( # SinePositionalEncoding
|
||||
num_feats=512, normalize=True),
|
||||
transformer_decoder=dict( # Mask2FormerTransformerDecoder
|
||||
return_intermediate=True,
|
||||
num_layers=9,
|
||||
layer_cfg=dict( # Mask2FormerTransformerDecoderLayer
|
||||
self_attn_cfg=dict( # MultiheadAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
attn_drop=0.0,
|
||||
proj_drop=0.0,
|
||||
dropout_layer=None,
|
||||
batch_first=True),
|
||||
cross_attn_cfg=dict( # MultiheadAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
attn_drop=0.0,
|
||||
proj_drop=0.0,
|
||||
dropout_layer=None,
|
||||
batch_first=True),
|
||||
ffn_cfg=dict(
|
||||
embed_dims=1024,
|
||||
feedforward_channels=4096,
|
||||
num_fcs=2,
|
||||
act_cfg=dict(type='ReLU', inplace=True),
|
||||
ffn_drop=0.0,
|
||||
dropout_layer=None,
|
||||
add_identity=True)),
|
||||
init_cfg=None),
|
||||
loss_cls=dict(
|
||||
type='mmdet.CrossEntropyLoss',
|
||||
use_sigmoid=False,
|
||||
loss_weight=2.0,
|
||||
reduction='mean',
|
||||
class_weight=[1.0] * num_classes + [0.1]),
|
||||
loss_mask=dict(
|
||||
type='mmdet.CrossEntropyLoss',
|
||||
use_sigmoid=True,
|
||||
reduction='mean',
|
||||
loss_weight=5.0),
|
||||
loss_dice=dict(
|
||||
type='mmdet.DiceLoss',
|
||||
use_sigmoid=True,
|
||||
activate=True,
|
||||
reduction='mean',
|
||||
naive_dice=True,
|
||||
eps=1.0,
|
||||
loss_weight=5.0),
|
||||
train_cfg=dict(
|
||||
num_points=12544,
|
||||
oversample_ratio=3.0,
|
||||
importance_sample_ratio=0.75,
|
||||
assigner=dict(
|
||||
type='mmdet.HungarianAssigner',
|
||||
match_costs=[
|
||||
dict(type='mmdet.ClassificationCost', weight=2.0),
|
||||
dict(
|
||||
type='mmdet.CrossEntropyLossCost',
|
||||
weight=5.0,
|
||||
use_sigmoid=True),
|
||||
dict(
|
||||
type='mmdet.DiceCost',
|
||||
weight=5.0,
|
||||
pred_act=True,
|
||||
eps=1.0)
|
||||
]),
|
||||
sampler=dict(type='mmdet.MaskPseudoSampler'))),
|
||||
train_cfg=dict(),
|
||||
test_cfg=dict(mode='slide', crop_size=crop_size, stride=(518, 518)))
|
||||
|
||||
# dataset config
|
||||
train_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='LoadAnnotations'),
|
||||
dict(
|
||||
type='RandomChoiceResize',
|
||||
scales=[int(x * 0.1 * 896) for x in range(5, 21)],
|
||||
resize_type='ResizeShortestEdge',
|
||||
max_size=896 * 4),
|
||||
dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75),
|
||||
dict(type='RandomFlip', prob=0.5),
|
||||
dict(type='PhotoMetricDistortion'),
|
||||
dict(type='PackSegInputs')
|
||||
]
|
||||
test_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='Resize', scale=(896 * 4, 896), keep_ratio=True),
|
||||
# add loading annotation after ``Resize`` because ground truth
|
||||
# does not need to do resize data transform
|
||||
dict(type='LoadAnnotations'),
|
||||
dict(type='PackSegInputs')
|
||||
]
|
||||
train_dataloader = dict(batch_size=1, dataset=dict(pipeline=train_pipeline))
|
||||
val_dataloader = dict(dataset=dict(pipeline=test_pipeline))
|
||||
test_dataloader = val_dataloader
|
||||
|
||||
# set all layers in backbone to lr_mult=0.1
|
||||
# set all norm layers, position_embeding,
|
||||
# query_embeding, level_embeding to decay_multi=0.0
|
||||
backbone_norm_multi = dict(lr_mult=0.1, decay_mult=0.0)
|
||||
backbone_embed_multi = dict(lr_mult=0.1, decay_mult=0.0)
|
||||
embed_multi = dict(lr_mult=1.0, decay_mult=0.0)
|
||||
custom_keys = {
|
||||
'backbone.dinov2': dict(lr_mult=0.1, decay_mult=1.0),
|
||||
'backbone.dinov2.norm': backbone_norm_multi,
|
||||
'pos_embed': backbone_embed_multi,
|
||||
'query_embed': embed_multi,
|
||||
'query_feat': embed_multi,
|
||||
'level_embed': embed_multi
|
||||
}
|
||||
custom_keys.update({
|
||||
f'backbone.dinov2.blocks.{block_id}.norm': backbone_norm_multi
|
||||
for block_id in range(24)
|
||||
})
|
||||
# optimizer
|
||||
optimizer = dict(
|
||||
type='AdamW', lr=0.00003, weight_decay=0.05, eps=1e-8, betas=(0.9, 0.999))
|
||||
optim_wrapper = dict(
|
||||
type='OptimWrapper',
|
||||
optimizer=optimizer,
|
||||
clip_grad=dict(max_norm=0.01, norm_type=2),
|
||||
paramwise_cfg=dict(custom_keys=custom_keys, norm_decay_mult=0.0))
|
||||
|
||||
find_unused_parameters=True
|
||||
|
||||
param_scheduler = [
|
||||
dict(
|
||||
type='LinearLR', start_factor=1e-6, by_epoch=False, begin=0, end=1500),
|
||||
dict(
|
||||
type='PolyLR',
|
||||
power=1.0,
|
||||
begin=1500,
|
||||
end=80000,
|
||||
eta_min=0.0,
|
||||
by_epoch=False,
|
||||
)
|
||||
]
|
||||
|
||||
# training schedule for 160k
|
||||
train_cfg = dict(
|
||||
type='IterBasedTrainLoop', max_iters=80000, val_interval=5000)
|
||||
val_cfg = dict(type='ValLoop')
|
||||
test_cfg = dict(type='TestLoop')
|
||||
default_hooks = dict(
|
||||
timer=dict(type='IterTimerHook'),
|
||||
logger=dict(type='LoggerHook', interval=50, log_metric_by_epoch=False),
|
||||
param_scheduler=dict(type='ParamSchedulerHook'),
|
||||
checkpoint=dict(
|
||||
type='CheckpointHook', by_epoch=False, interval=5000, save_best='mIoU', max_keep_ckpts=1),
|
||||
sampler_seed=dict(type='DistSamplerSeedHook'),
|
||||
visualization=dict(type='SegVisualizationHook'))
|
||||
|
||||
# Default setting for scaling LR automatically
|
||||
# - `enable` means enable scaling LR automatically
|
||||
# or not by default.
|
||||
# - `base_batch_size` = (8 GPUs) x (2 samples per GPU).
|
||||
auto_scale_lr = dict(enable=False, base_batch_size=16)
|
||||
|
||||
work_dir = './work_dirs/depth_anything_large_mask2former_16xb1_80k_cityscapes_896x896'
|
||||
@@ -0,0 +1,244 @@
|
||||
_base_ = [
|
||||
'../_base_/default_runtime.py', '../_base_/datasets/cityscapes.py'
|
||||
]
|
||||
|
||||
crop_size = (896, 896)
|
||||
data_preprocessor = dict(
|
||||
type='SegDataPreProcessor',
|
||||
mean=[123.675, 116.28, 103.53],
|
||||
std=[58.395, 57.12, 57.375],
|
||||
bgr_to_rgb=True,
|
||||
pad_val=0,
|
||||
seg_pad_val=255,
|
||||
size=crop_size,
|
||||
test_cfg=dict(size_divisor=14))
|
||||
num_classes = 19
|
||||
|
||||
model = dict(
|
||||
type='EncoderDecoder',
|
||||
data_preprocessor=data_preprocessor,
|
||||
backbone=dict(
|
||||
type='DINOv2',
|
||||
version='large',
|
||||
freeze=False,
|
||||
load_from='./checkpoints/depth_anything_vitl14.pth'),
|
||||
neck=dict(type='Feature2Pyramid', embed_dim=1024, rescales=[4, 2, 1, 0.5]),
|
||||
decode_head=dict(
|
||||
type='Mask2FormerHead',
|
||||
in_channels=[1024, 1024, 1024, 1024],
|
||||
# strides=[4, 8, 16, 32],
|
||||
feat_channels=1024,
|
||||
out_channels=1024,
|
||||
num_classes=num_classes,
|
||||
num_queries=200,
|
||||
num_transformer_feat_level=3,
|
||||
align_corners=False,
|
||||
pixel_decoder=dict(
|
||||
type='mmdet.MSDeformAttnPixelDecoder',
|
||||
num_outs=3,
|
||||
norm_cfg=dict(type='GN', num_groups=32),
|
||||
act_cfg=dict(type='ReLU'),
|
||||
encoder=dict( # DeformableDetrTransformerEncoder
|
||||
num_layers=6,
|
||||
layer_cfg=dict( # DeformableDetrTransformerEncoderLayer
|
||||
self_attn_cfg=dict( # MultiScaleDeformableAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
num_levels=3,
|
||||
num_points=4,
|
||||
im2col_step=64,
|
||||
dropout=0.0,
|
||||
batch_first=True,
|
||||
norm_cfg=None,
|
||||
init_cfg=None),
|
||||
ffn_cfg=dict(
|
||||
embed_dims=1024,
|
||||
feedforward_channels=4096,
|
||||
num_fcs=2,
|
||||
ffn_drop=0.0,
|
||||
act_cfg=dict(type='ReLU', inplace=True))),
|
||||
init_cfg=None),
|
||||
positional_encoding=dict( # SinePositionalEncoding
|
||||
num_feats=512, normalize=True),
|
||||
init_cfg=None),
|
||||
enforce_decoder_input_project=False,
|
||||
positional_encoding=dict( # SinePositionalEncoding
|
||||
num_feats=512, normalize=True),
|
||||
transformer_decoder=dict( # Mask2FormerTransformerDecoder
|
||||
return_intermediate=True,
|
||||
num_layers=9,
|
||||
layer_cfg=dict( # Mask2FormerTransformerDecoderLayer
|
||||
self_attn_cfg=dict( # MultiheadAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
attn_drop=0.0,
|
||||
proj_drop=0.0,
|
||||
dropout_layer=None,
|
||||
batch_first=True),
|
||||
cross_attn_cfg=dict( # MultiheadAttention
|
||||
embed_dims=1024,
|
||||
num_heads=32,
|
||||
attn_drop=0.0,
|
||||
proj_drop=0.0,
|
||||
dropout_layer=None,
|
||||
batch_first=True),
|
||||
ffn_cfg=dict(
|
||||
embed_dims=1024,
|
||||
feedforward_channels=4096,
|
||||
num_fcs=2,
|
||||
act_cfg=dict(type='ReLU', inplace=True),
|
||||
ffn_drop=0.0,
|
||||
dropout_layer=None,
|
||||
add_identity=True)),
|
||||
init_cfg=None),
|
||||
loss_cls=dict(
|
||||
type='mmdet.CrossEntropyLoss',
|
||||
use_sigmoid=False,
|
||||
loss_weight=2.0,
|
||||
reduction='mean',
|
||||
class_weight=[1.0] * num_classes + [0.1]),
|
||||
loss_mask=dict(
|
||||
type='mmdet.CrossEntropyLoss',
|
||||
use_sigmoid=True,
|
||||
reduction='mean',
|
||||
loss_weight=5.0),
|
||||
loss_dice=dict(
|
||||
type='mmdet.DiceLoss',
|
||||
use_sigmoid=True,
|
||||
activate=True,
|
||||
reduction='mean',
|
||||
naive_dice=True,
|
||||
eps=1.0,
|
||||
loss_weight=5.0),
|
||||
train_cfg=dict(
|
||||
num_points=12544,
|
||||
oversample_ratio=3.0,
|
||||
importance_sample_ratio=0.75,
|
||||
assigner=dict(
|
||||
type='mmdet.HungarianAssigner',
|
||||
match_costs=[
|
||||
dict(type='mmdet.ClassificationCost', weight=2.0),
|
||||
dict(
|
||||
type='mmdet.CrossEntropyLossCost',
|
||||
weight=5.0,
|
||||
use_sigmoid=True),
|
||||
dict(
|
||||
type='mmdet.DiceCost',
|
||||
weight=5.0,
|
||||
pred_act=True,
|
||||
eps=1.0)
|
||||
]),
|
||||
sampler=dict(type='mmdet.MaskPseudoSampler'))),
|
||||
train_cfg=dict(),
|
||||
test_cfg=dict(mode='slide', crop_size=crop_size, stride=(518, 518)))
|
||||
|
||||
# dataset config
|
||||
img_norm_cfg = dict(
|
||||
mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True)
|
||||
train_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='LoadAnnotations'),
|
||||
dict(
|
||||
type='RandomChoiceResize',
|
||||
scales=[int(x * 0.1 * 896) for x in range(5, 21)],
|
||||
resize_type='ResizeShortestEdge',
|
||||
max_size=896 * 4),
|
||||
dict(type='RandomCrop', crop_size=crop_size, cat_max_ratio=0.75),
|
||||
dict(type='RandomFlip', prob=0.5),
|
||||
dict(type='PhotoMetricDistortion'),
|
||||
dict(type='PackSegInputs')
|
||||
]
|
||||
test_pipeline = [
|
||||
dict(type='LoadImageFromFile'),
|
||||
dict(type='Resize', scale=(896 * 4, 896), keep_ratio=True),
|
||||
# add loading annotation after ``Resize`` because ground truth
|
||||
# does not need to do resize data transform
|
||||
dict(type='LoadAnnotations'),
|
||||
dict(type='PackSegInputs')
|
||||
]
|
||||
|
||||
img_ratios = [0.5, 0.75, 1.0, 1.25, 1.5, 1.75, 2.0]
|
||||
# img_ratios = [1.0]
|
||||
tta_pipeline = [
|
||||
dict(type='LoadImageFromFile', backend_args=None),
|
||||
dict(
|
||||
type='TestTimeAug',
|
||||
transforms=[
|
||||
[
|
||||
dict(type='Resize', scale_factor=r, keep_ratio=True)
|
||||
for r in img_ratios
|
||||
],
|
||||
[
|
||||
dict(type='RandomFlip', prob=0., direction='horizontal'),
|
||||
dict(type='RandomFlip', prob=1., direction='horizontal')
|
||||
], [dict(type='LoadAnnotations')], [dict(type='PackSegInputs')]
|
||||
])
|
||||
]
|
||||
|
||||
train_dataloader = dict(batch_size=1, dataset=dict(pipeline=train_pipeline))
|
||||
val_dataloader = dict(dataset=dict(pipeline=test_pipeline))
|
||||
test_dataloader = val_dataloader
|
||||
|
||||
# set all layers in backbone to lr_mult=0.1
|
||||
# set all norm layers, position_embeding,
|
||||
# query_embeding, level_embeding to decay_multi=0.0
|
||||
backbone_norm_multi = dict(lr_mult=0.1, decay_mult=0.0)
|
||||
backbone_embed_multi = dict(lr_mult=0.1, decay_mult=0.0)
|
||||
embed_multi = dict(lr_mult=1.0, decay_mult=0.0)
|
||||
custom_keys = {
|
||||
'backbone.dinov2': dict(lr_mult=0.1, decay_mult=1.0),
|
||||
'backbone.dinov2.norm': backbone_norm_multi,
|
||||
'pos_embed': backbone_embed_multi,
|
||||
'query_embed': embed_multi,
|
||||
'query_feat': embed_multi,
|
||||
'level_embed': embed_multi
|
||||
}
|
||||
custom_keys.update({
|
||||
f'backbone.dinov2.blocks.{block_id}.norm': backbone_norm_multi
|
||||
for block_id in range(24)
|
||||
})
|
||||
# optimizer
|
||||
optimizer = dict(
|
||||
type='AdamW', lr=0.00003, weight_decay=0.05, eps=1e-8, betas=(0.9, 0.999))
|
||||
optim_wrapper = dict(
|
||||
type='OptimWrapper',
|
||||
optimizer=optimizer,
|
||||
clip_grad=dict(max_norm=0.01, norm_type=2),
|
||||
paramwise_cfg=dict(custom_keys=custom_keys, norm_decay_mult=0.0))
|
||||
|
||||
find_unused_parameters=True
|
||||
|
||||
param_scheduler = [
|
||||
dict(
|
||||
type='LinearLR', start_factor=1e-6, by_epoch=False, begin=0, end=1500),
|
||||
dict(
|
||||
type='PolyLR',
|
||||
power=1.0,
|
||||
begin=1500,
|
||||
end=80000,
|
||||
eta_min=0.0,
|
||||
by_epoch=False,
|
||||
)
|
||||
]
|
||||
|
||||
# training schedule for 160k
|
||||
train_cfg = dict(
|
||||
type='IterBasedTrainLoop', max_iters=80000, val_interval=5000)
|
||||
val_cfg = dict(type='ValLoop')
|
||||
test_cfg = dict(type='TestLoop')
|
||||
default_hooks = dict(
|
||||
timer=dict(type='IterTimerHook'),
|
||||
logger=dict(type='LoggerHook', interval=1, log_metric_by_epoch=False),
|
||||
param_scheduler=dict(type='ParamSchedulerHook'),
|
||||
checkpoint=dict(
|
||||
type='CheckpointHook', by_epoch=False, interval=5000, save_best='mIoU', max_keep_ckpts=1),
|
||||
sampler_seed=dict(type='DistSamplerSeedHook'),
|
||||
visualization=dict(type='SegVisualizationHook'))
|
||||
|
||||
# Default setting for scaling LR automatically
|
||||
# - `enable` means enable scaling LR automatically
|
||||
# or not by default.
|
||||
# - `base_batch_size` = (8 GPUs) x (2 samples per GPU).
|
||||
auto_scale_lr = dict(enable=False, base_batch_size=16)
|
||||
|
||||
work_dir = './work_dirs/depth_anything_large_mask2former_16xb1_80k_cityscapes_896x896_ms'
|
||||
47
Depth-Anything-V1-main/semseg/dinov2.py
Normal file
47
Depth-Anything-V1-main/semseg/dinov2.py
Normal file
@@ -0,0 +1,47 @@
|
||||
import torch
|
||||
from mmengine.model import BaseModule
|
||||
from torch import nn
|
||||
|
||||
from mmseg.registry import MODELS
|
||||
|
||||
|
||||
|
||||
@MODELS.register_module()
|
||||
class DINOv2(nn.Module):
|
||||
"""Use DINOv2 pre-trained models
|
||||
"""
|
||||
|
||||
def __init__(self, version='large', freeze=False, load_from=None):
|
||||
super().__init__()
|
||||
|
||||
if version == 'large':
|
||||
self.dinov2 = torch.hub.load('torchhub/facebookresearch_dinov2_main', 'dinov2_vitl14', source='local', pretrained=False)
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
if load_from is not None:
|
||||
d = torch.load(load_from, map_location='cpu')
|
||||
new_d = {}
|
||||
for key, value in d.items():
|
||||
if 'pretrained' in key:
|
||||
new_d[key.replace('pretrained.', '')] = value
|
||||
self.dinov2.load_state_dict(new_d)
|
||||
|
||||
self.freeze = freeze
|
||||
|
||||
def forward(self, inputs):
|
||||
B, _, h, w = inputs.shape
|
||||
|
||||
if self.freeze:
|
||||
with torch.no_grad():
|
||||
features = self.dinov2.get_intermediate_layers(inputs, 4)
|
||||
else:
|
||||
features = self.dinov2.get_intermediate_layers(inputs, 4)
|
||||
|
||||
outs = []
|
||||
for feature in features:
|
||||
C = feature.shape[-1]
|
||||
feature = feature.permute(0, 2, 1).reshape(B, C, h // 14, w // 14).contiguous()
|
||||
outs.append(feature)
|
||||
|
||||
return outs
|
||||
Reference in New Issue
Block a user