Merge pull request #618 from open-mmlab/dev-1.x

Merge dev-1.x to 1.x
open-mmlab · Dec 7, 2022 · 0d6632a · 0d6632a
2 parents 7980414 + c11ac99
commit 0d6632a
Show file tree

Hide file tree

Showing 143 changed files with 5,969 additions and 986 deletions.
diff --git a/.github/workflows/merge_stage_test.yml b/.github/workflows/merge_stage_test.yml
@@ -9,6 +9,7 @@ on:
       - 'demo/**'
       - '.dev_scripts/**'
       - '.circleci/**'
+      - 'projects/**'
     branches:
       - dev-1.x
 
@@ -164,7 +165,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Upgrade pip
-        run: pip install pip --upgrade
+        run: python.exe -m pip install --upgrade pip
       - name: Install lmdb
         run: pip install lmdb
       - name: Install PyTorch

diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml
@@ -9,6 +9,7 @@ on:
       - 'demo/**'
       - '.dev_scripts/**'
       - '.circleci/**'
+      - 'projects/**'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -98,6 +99,58 @@ jobs:
           python setup.py check -m -s
           TORCH_CUDA_ARCH_LIST=7.0 pip install -e .
 
+  build_cu116:
+    runs-on: ubuntu-18.04
+    container:
+      image: pytorch/pytorch:1.13.0-cuda11.6-cudnn8-devel
+    env:
+      FORCE_CUDA: 1
+      MMCV_CUDA_ARGS: -gencode=arch=compute_61,code=sm_61
+    strategy:
+      matrix:
+        python-version: ['3.9']
+        torch: [1.13.0+cu116]
+        include:
+          - torch: 1.13.0+cu116
+            torchvision: 0.14.0+cu116
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Fetch GPG keys
+        run: |
+          apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
+          apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
+      - name: Add PPA
+        run: |
+          apt-get update && apt-get install -y software-properties-common
+          add-apt-repository -y ppa:deadsnakes/ppa
+      - name: Install python-dev
+        run: apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y python${{matrix.python-version}}-dev
+      - name: python -m Install PyTorch
+        run: python -m pip install torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
+      - name: Install system dependencies
+        run: apt-get update && apt-get install -y ffmpeg libturbojpeg ninja-build
+      - name: Install MMSelfSup dependencies
+        run: |
+          pip install -U openmim
+          mim install mmengine
+          mim install 'mmcv>=2.0.0rc1'
+          mim install 'mmdet>=3.0.0rc0'
+          mim install 'mmsegmentation>=1.0.0rc0'
+          pip install -r requirements/tests.txt
+      - name: Build and install
+        run: |
+          rm -rf .eggs
+          pip install -e . -U
+      - name: Run unittests and generate coverage report
+        run: |
+          coverage run --branch --source mmselfsup -m pytest tests/
+          coverage xml
+          coverage report -m
+
   build_windows:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -112,7 +165,7 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
       - name: Upgrade pip
-        run: pip install pip --upgrade
+        run: python.exe -m pip install --upgrade pip
       - name: Install lmdb
         run: pip install lmdb
       - name: Install PyTorch

diff --git a/.gitignore b/.gitignore
@@ -64,8 +64,14 @@ instance/
 # Scrapy stuff:
 .scrapy
 
-# Sphinx documentation
-docs/_build/
+# Sphinx documentation (readthedocs)
+src/
+docs/en/_build/
+docs/en/model_zoo_statistics.md
+docs/en/papers/
+docs/zh_cn/_build/
+docs/zh_cn/model_zoo_statistics.md
+docs/zh_cn/papers/
 
 # PyBuilder
 target/
@@ -125,9 +131,3 @@ INFO
 
 # Pytorch
 *.pth
-
-
-# readthedocs
-docs/zh_cn/_build
-src/
-docs/en/_build
diff --git a/README.md b/README.md
@@ -65,15 +65,15 @@ The master branch works with **PyTorch 1.6** or higher.
 
 ## What's New
 
-MMSelfSup **v1.0.0rc3** was released in 01/11/2022.
+MMSelfSup **v1.0.0rc4** was released in 07/12/2022.
 
-- Support MaskFeat.
-- Refine configs and documents.
+- Support BEiT v1 and MILAN.
+- Support low-level reconstruction visualization.
+- Refine documents algorithm pages.
 
-MMSelfSup **v1.0.0rc2** was released in 12/10/2022.
+MMSelfSup **v1.0.0rc3** was released in 01/11/2022.
 
-- Full support of `MAE`, `SimMIM`, `MoCoV3` with different model sizes and different pre-training epochs.
-- Update documents and colab tutorial.
+- Support MaskFeat.
 
 Please refer to [Changelog](https://mmselfsup.readthedocs.io/en/dev-1.x/notes/changelog.html) for details and release history.
 
@@ -130,10 +130,12 @@ Supported algorithms:
 - [x] [SimSiam (CVPR'2021)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/simsiam)
 - [x] [Barlow Twins (ICML'2021)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/barlowtwins)
 - [x] [MoCo v3 (ICCV'2021)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/mocov3)
+- [x] [BEiT (ICLR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/beit)
 - [x] [MAE (CVPR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/mae)
 - [x] [SimMIM (CVPR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/simmim)
 - [x] [MaskFeat (CVPR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/maskfeat)
 - [x] [CAE (ArXiv'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/cae)
+- [x] [MILAN (ArXiv'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/milan)
 
 More algorithms are in our plan.
 

diff --git a/README_zh-CN.md b/README_zh-CN.md
@@ -65,15 +65,15 @@ MMSelfSup 是一个基于 PyTorch 实现的开源自监督表征学习工具箱
 
 ## 更新
 
-**v1.0.0rc3** 版本已经在 2022.11.01 发布。
+**v1.0.0rc3** 版本已经在 2022.12.07 发布。
 
-- 支持 MaskFeat
-- 更新配置文件和文档
+- 支持 BEiT 和 MILAN 预训练
+- 支持 low-level 重建可视化
+- 更新模型库算法页面文档
 
-**v1.0.0rc2** 版本已经在 2022.10.12 发布。
+**v1.0.0rc3** 版本已经在 2022.11.01 发布。
 
-- `MAE`, `SimMIM`, `MoCoV3` 算法模型的全量支持（支持多种模型尺寸和预训练轮次）。
-- 更新文档以及教程
+- 支持 MaskFeat
 
 请参考 [更新日志](https://mmselfsup.readthedocs.io/zh_CN/dev-1.x/notes/changelog.html) 获取更多细节和历史版本信息。
 
@@ -130,10 +130,12 @@ Useful Tools
 - [x] [SimSiam (CVPR'2021)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/simsiam)
 - [x] [Barlow Twins (ICML'2021)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/barlowtwins)
 - [x] [MoCo v3 (ICCV'2021)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/mocov3)
+- [x] [BEiT (ICLR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/beit)
 - [x] [MAE (CVPR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/mae)
 - [x] [SimMIM (CVPR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/simmim)
 - [x] [MaskFeat (CVPR'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/maskfeat)
 - [x] [CAE (ArXiv'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/cae)
+- [x] [MILAN (ArXiv'2022)](https://github.com/open-mmlab/mmselfsup/tree/dev-1.x/configs/selfsup/milan)
 
 更多的算法实现已经在我们的计划中。
 

diff --git a/configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-192.py b/configs/benchmarks/classification/imagenet/swin-base_ft-8xb256-coslr-100e_in1k-192.py
@@ -19,14 +19,14 @@
     optimizer=dict(
         type='AdamW', lr=5e-3, model_type='swin', layer_decay_rate=0.9),
     clip_grad=dict(max_norm=5.0),
+    constructor='mmselfsup.LearningRateDecayOptimWrapperConstructor',
     paramwise_cfg=dict(
-        norm_decay_mult=0.0,
-        bias_decay_mult=0.0,
         custom_keys={
+            '.norm': dict(decay_mult=0.0),
+            '.bias': dict(decay_mult=0.0),
             '.absolute_pos_embed': dict(decay_mult=0.0),
             '.relative_position_bias_table': dict(decay_mult=0.0)
-        }),
-    constructor='mmselfsup.LearningRateDecayOptimWrapperConstructor')
+        }))
 
 # learning rate scheduler
 param_scheduler = [

diff --git a/configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k.py b/configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e-rpe_in1k.py
@@ -97,7 +97,14 @@
         weight_decay=0.05,
         model_type='vit',  # layer-wise lr decay type
         layer_decay_rate=0.65),  # layer-wise lr decay factor
-    constructor='mmselfsup.LearningRateDecayOptimWrapperConstructor')
+    constructor='mmselfsup.LearningRateDecayOptimWrapperConstructor',
+    paramwise_cfg=dict(
+        custom_keys={
+            '.ln': dict(decay_mult=0.0),
+            '.bias': dict(decay_mult=0.0),
+            '.cls_token': dict(decay_mult=0.0),
+            '.pos_embed': dict(decay_mult=0.0)
+        }))
 
 # learning rate scheduler
 param_scheduler = [

diff --git a/configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e_in1k.py b/configs/benchmarks/classification/imagenet/vit-base-p16_ft-8xb128-coslr-100e_in1k.py
@@ -88,9 +88,9 @@
         layer_decay_rate=0.65),  # layer-wise lr decay factor
     constructor='mmselfsup.LearningRateDecayOptimWrapperConstructor',
     paramwise_cfg=dict(
-        norm_decay_mult=0.0,
-        bias_decay_mult=0.0,
         custom_keys={
+            '.ln': dict(decay_mult=0.0),
+            '.bias': dict(decay_mult=0.0),
             '.cls_token': dict(decay_mult=0.0),
             '.pos_embed': dict(decay_mult=0.0)
         }))
@@ -121,4 +121,4 @@
 
 train_cfg = dict(by_epoch=True, max_epochs=100)
 
-randomness = dict(seed=0)
+randomness = dict(seed=0, diff_rank_seed=True)
diff --git a/configs/benchmarks/classification/imagenet/vit-base-p16_linear-8xb2048-coslr-90e_in1k.py b/configs/benchmarks/classification/imagenet/vit-base-p16_linear-8xb2048-coslr-90e_in1k.py
@@ -46,9 +46,10 @@
     dict(type='CenterCrop', crop_size=224),
     dict(type='PackClsInputs'),
 ]
-train_dataloader = dict(batch_size=2048, dataset=dict(pipeline=train_pipeline))
-val_dataloader = dict(dataset=dict(pipeline=test_pipeline))
-test_dataloader = dict(dataset=dict(pipeline=test_pipeline))
+train_dataloader = dict(
+    batch_size=2048, dataset=dict(pipeline=train_pipeline), drop_last=True)
+val_dataloader = dict(dataset=dict(pipeline=test_pipeline), drop_last=False)
+test_dataloader = dict(dataset=dict(pipeline=test_pipeline), drop_last=False)
 
 # optimizer
 optimizer = dict(type='mmselfsup.LARS', lr=6.4, weight_decay=0.0, momentum=0.9)

diff --git a/configs/selfsup/_base_/datasets/imagenet_beit.py b/configs/selfsup/_base_/datasets/imagenet_beit.py
@@ -0,0 +1,52 @@
+# dataset settings
+dataset_type = 'mmcls.ImageNet'
+data_root = 'data/imagenet/'
+file_client_args = dict(backend='disk')
+train_pipeline = [
+    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(
+        type='ColorJitter',
+        brightness=0.4,
+        contrast=0.4,
+        saturation=0.4,
+        hue=0.),
+    dict(type='RandomFlip', prob=0.5, direction='horizontal'),
+    dict(
+        type='RandomResizedCropAndInterpolationWithTwoPic',
+        size=224,
+        second_size=112,
+        interpolation='bicubic',
+        second_interpolation='lanczos',
+        scale=(0.08, 1.0)),
+    dict(
+        type='BEiTMaskGenerator',
+        input_size=(14, 14),
+        num_masking_patches=75,
+        max_num_patches=None,
+        min_num_patches=16),
+    dict(
+        type='PackSelfSupInputs',
+        algorithm_keys=['mask'],
+        meta_keys=['img_path'])
+]
+
+data_preprocessor = dict(
+    type='TwoNormDataPreprocessor',
+    mean=(123.675, 116.28, 103.53),
+    std=(58.395, 57.12, 57.375),
+    second_mean=(-20.4, -20.4, -20.4),
+    second_std=(204., 204., 204.),
+    bgr_to_rgb=True)
+
+train_dataloader = dict(
+    batch_size=256,
+    num_workers=8,
+    persistent_workers=True,
+    sampler=dict(type='DefaultSampler', shuffle=True),
+    collate_fn=dict(type='default_collate'),
+    dataset=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file='meta/train.txt',
+        data_prefix=dict(img_path='train/'),
+        pipeline=train_pipeline))
diff --git a/configs/selfsup/_base_/datasets/imagenet_beitv2.py b/configs/selfsup/_base_/datasets/imagenet_beitv2.py
@@ -0,0 +1,52 @@
+# dataset settings
+dataset_type = 'mmcls.ImageNet'
+data_root = 'data/imagenet/'
+file_client_args = dict(backend='disk')
+train_pipeline = [
+    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(
+        type='ColorJitter',
+        brightness=0.4,
+        contrast=0.4,
+        saturation=0.4,
+        hue=0.),
+    dict(type='RandomFlip', prob=0.5, direction='horizontal'),
+    dict(
+        type='RandomResizedCropAndInterpolationWithTwoPic',
+        size=224,
+        second_size=224,
+        interpolation='bicubic',
+        second_interpolation='bicubic',
+        scale=(0.2, 1.0)),
+    dict(
+        type='BEiTMaskGenerator',
+        input_size=(14, 14),
+        num_masking_patches=75,
+        max_num_patches=75,
+        min_num_patches=16),
+    dict(
+        type='PackSelfSupInputs',
+        algorithm_keys=['mask'],
+        meta_keys=['img_path'])
+]
+
+data_preprocessor = dict(
+    type='TwoNormDataPreprocessor',
+    mean=(123.675, 116.28, 103.53),
+    std=(58.395, 57.12, 57.375),
+    second_mean=(127.5, 127.5, 127.5),
+    second_std=(127.5, 127.5, 127.5),
+    bgr_to_rgb=True)
+
+train_dataloader = dict(
+    batch_size=256,
+    num_workers=8,
+    persistent_workers=True,
+    sampler=dict(type='DefaultSampler', shuffle=True),
+    collate_fn=dict(type='default_collate'),
+    dataset=dict(
+        type=dataset_type,
+        data_root=data_root,
+        ann_file='meta/train.txt',
+        data_prefix=dict(img_path='train/'),
+        pipeline=train_pipeline))
diff --git a/configs/selfsup/_base_/datasets/imagenet_maskfeat.py b/configs/selfsup/_base_/datasets/imagenet_maskfeat.py
@@ -37,3 +37,19 @@
         ann_file='meta/train.txt',
         data_prefix=dict(img_path='train/'),
         pipeline=train_pipeline))
+
+# for visualization
+vis_pipeline = [
+    dict(type='LoadImageFromFile', file_client_args=file_client_args),
+    dict(type='Resize', scale=(224, 224), backend='pillow'),
+    dict(
+        type='BEiTMaskGenerator',
+        input_size=14,
+        num_masking_patches=78,
+        min_num_patches=15,
+    ),
+    dict(
+        type='PackSelfSupInputs',
+        algorithm_keys=['mask'],
+        meta_keys=['img_path'])
+]