Arch update 21/11 - let's simplify it a bit ;) (#1346)

* a few simplifications

* docs simplifications

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* fix

* contrib simplification

* fix

Author: Scitator

CHANGELOG.md

@@ -12,15 +12,36 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
--
+- A few framework simplifications were made ([#1346](https://github.com/catalyst-team/catalyst/pull/1346)):
+  - `catalyst-contrib` scripts reduced to `collect-env` and `project-embeddings` only
+  - `catalyst-dl` scripts recuded to `run` and `tune` only
+  - `transforms.` prefix deprecated for Catalyst-based transforms
+  - `catalyst.tools` moved to `catalyst.extras`
+  - task-dependent extensions from `catalyst.data` moved to `catalyst.contrib.data`
+  - `catalyst.data.transforms` moved to `catalyst.contrib.data.transforms`
+  - `Normalize`, `ToTensor` transforms renamed to `NormalizeImage`, `ImageToTensor`
+  - metric learning extensions moved to `catalyst.contrib.data`
+  - `catalyst.contrib` moved to code-as-a-documentation development
+  - `catalyst[cv]` and `catalyst[ml]` extensions moved to flatten architecture design; examples: `catalyst.contrib.data.dataset_cv`, `catalyst.contrib.data.dataset_ml`
+  - `catalyst.contrib` moved to flatten architecture design; exampels: `catalyst.contrib.data`, `catalyst.contrib.datasets`, `catalyst.contrib.layers`, `catalyst.contrib.models`, `catalyst.contrib.optimizers`, `catalyst.contrib.schedulers`
+  - internal functionality moved to `***._misc` modules
+  - `catalyst.utils.mixup` moved to `catalyst.utils.torch`
+  - `catalyst.utils.numpy` moved to `catalyst.contrib.utils.numpy`
+  
 
 ### Removed
 
--
+- A few framework simplifications were made ([#1346](https://github.com/catalyst-team/catalyst/pull/1346)):
+  - `catalyst.contrib.pandas`
+  - `catalyst.contrib.parallel`
+  - `catalyst.contrib.models.cv`
+  - a few `catalyst.utils.misc` functions
+  - `catalyst.extras` removed from the public documentation
+
 
 ### Fixed
 
--
+- documentation search error (21.10 only) ([#1346](https://github.com/catalyst-team/catalyst/pull/1346))
 
 
 ## [21.10] - 2021-10-30
@@ -498,7 +519,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Runner registry support for Config API ([#936](https://github.com/catalyst-team/catalyst/pull/936))
 - `catalyst-dl tune` command - Optuna with Config API integration for AutoML hyperparameters optimization ([#937](https://github.com/catalyst-team/catalyst/pull/937))
 - `OptunaPruningCallback` alias for `OptunaCallback` ([#937](https://github.com/catalyst-team/catalyst/pull/937))
-- AdamP and SGDP to `catalyst.contrib.nn.criterion` ([#942](https://github.com/catalyst-team/catalyst/pull/942))
+- AdamP and SGDP to `catalyst.contrib.losses` ([#942](https://github.com/catalyst-team/catalyst/pull/942))
 
 ### Changed
 

README.md

@@ -61,15 +61,14 @@ import os
 from torch import nn, optim
 from torch.utils.data import DataLoader
 from catalyst import dl, utils
-from catalyst.data import ToTensor
-from catalyst.contrib.datasets import MNIST
+from catalyst.contrib import ImageToTensor, MNIST
 
 model = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10))
 criterion = nn.CrossEntropyLoss()
 optimizer = optim.Adam(model.parameters(), lr=0.02)
 
-train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
 loaders = {
     "train": DataLoader(train_data, batch_size=32),
     "valid": DataLoader(valid_data, batch_size=32),
@@ -105,7 +104,7 @@ metrics = runner.evaluate_loader(
     loader=loaders["valid"],
     callbacks=[dl.AccuracyCallback(input_key="logits", target_key="targets", topk_args=(1, 3, 5))],
 )
-assert "accuracy" in metrics.keys()
+assert "accuracy01" in metrics.keys()
 
 # model inference
 for prediction in runner.predict_loader(loader=loaders["valid"]):
@@ -115,13 +114,13 @@ features_batch = next(iter(loaders["valid"]))[0]
 # model stochastic weight averaging
 model.load_state_dict(utils.get_averaged_weights_by_path_mask(logdir="./logs", path_mask="*.pth"))
 # model tracing
-utils.trace_model(model=runner.model, batch=features_batch)
+utils.trace_model(model=runner.model.cpu(), batch=features_batch)
 # model quantization
 utils.quantize_model(model=runner.model)
 # model pruning
 utils.prune_model(model=runner.model, pruning_fn="l1_unstructured", amount=0.8)
 # onnx export
-utils.onnx_export(model=runner.model, batch=features_batch, file="./logs/mnist.onnx", verbose=True)
+utils.onnx_export(model=runner.model.cpu(), batch=features_batch, file="./logs/mnist.onnx", verbose=True)
 ```
 
 ### Step-by-step Guide
@@ -248,14 +247,13 @@ from torch import nn, optim
 from torch.nn import functional as F
 from torch.utils.data import DataLoader
 from catalyst import dl, metrics
-from catalyst.data import ToTensor
-from catalyst.contrib.datasets import MNIST
+from catalyst.contrib import ImageToTensor, MNIST
 
 model = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10))
 optimizer = optim.Adam(model.parameters(), lr=0.02)
 
-train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
 loaders = {
     "train": DataLoader(train_data, batch_size=32),
     "valid": DataLoader(valid_data, batch_size=32),
@@ -325,7 +323,7 @@ for logits in runner.predict_loader(loader=loaders["valid"]):
 <p>
 
 ```python
-import torch
+import torchx
 from torch.utils.data import DataLoader, TensorDataset
 from catalyst import dl
 
@@ -651,15 +649,14 @@ import os
 from torch import nn, optim
 from torch.utils.data import DataLoader
 from catalyst import dl
-from catalyst.data import ToTensor
-from catalyst.contrib.datasets import MNIST
+from catalyst.contrib import ImageToTensor, MNIST
 
 model = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10))
 criterion = nn.CrossEntropyLoss()
 optimizer = optim.Adam(model.parameters(), lr=0.02)
 
-train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
 loaders = {
     "train": DataLoader(train_data, batch_size=32),
     "valid": DataLoader(valid_data, batch_size=32),
@@ -706,9 +703,7 @@ import torch
 from torch import nn
 from torch.utils.data import DataLoader
 from catalyst import dl
-from catalyst.data import ToTensor
-from catalyst.contrib.datasets import MNIST
-from catalyst.contrib.nn import IoULoss
+from catalyst.contrib import ImageToTensor, IoULoss, MNIST
 
 
 model = nn.Sequential(
@@ -718,8 +713,8 @@ model = nn.Sequential(
 criterion = IoULoss()
 optimizer = torch.optim.Adam(model.parameters(), lr=0.02)
 
-train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
 loaders = {
     "train": DataLoader(train_data, batch_size=32),
     "valid": DataLoader(valid_data, batch_size=32),
@@ -769,17 +764,16 @@ from torch import nn, optim
 from torch.nn import functional as F
 from torch.utils.data import DataLoader
 from catalyst import dl
-from catalyst.data import ToTensor
-from catalyst.contrib.datasets import MNIST
+from catalyst.contrib import ImageToTensor, MNIST
 
 # [!] teacher model should be already pretrained
 teacher = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10))
 student = nn.Sequential(nn.Flatten(), nn.Linear(28 * 28, 10))
 criterion = {"cls": nn.CrossEntropyLoss(), "kl": nn.KLDivLoss(reduction="batchmean")}
 optimizer = optim.Adam(student.parameters(), lr=0.02)
 
-train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
 loaders = {
     "train": DataLoader(train_data, batch_size=32),
     "valid": DataLoader(valid_data, batch_size=32),
@@ -801,23 +795,13 @@ class DistilRunner(dl.Runner):
 
 runner = DistilRunner()
 callbacks = [
-    dl.AccuracyCallback(
-        input_key="t_logits", target_key="targets", num_classes=2, prefix="teacher_"
-    ),
-    dl.AccuracyCallback(
-        input_key="s_logits", target_key="targets", num_classes=2, prefix="student_"
-    ),
-    dl.CriterionCallback(
-        input_key="s_logits", target_key="targets", metric_key="cls_loss", criterion_key="cls"
-    ),
-    dl.CriterionCallback(
-        input_key="s_logprobs", target_key="t_probs", metric_key="kl_div_loss", criterion_key="kl"
-    ),
+    dl.AccuracyCallback(input_key="t_logits", target_key="targets", num_classes=2, prefix="teacher_"),
+    dl.AccuracyCallback(input_key="s_logits", target_key="targets", num_classes=2, prefix="student_"),
+    dl.CriterionCallback(input_key="s_logits", target_key="targets", metric_key="cls_loss", criterion_key="cls"),
+    dl.CriterionCallback(input_key="s_logprobs", target_key="t_probs", metric_key="kl_div_loss", criterion_key="kl"),
     dl.MetricAggregationCallback(metric_key="loss", metrics=["kl_div_loss", "cls_loss"], mode="mean"),
     dl.OptimizerCallback(metric_key="loss", model_key="student"),
-    dl.CheckpointCallback(
-        logdir="./logs", loader_key="valid", metric_key="loss", minimize=True, save_n_best=3
-    ),
+    dl.CheckpointCallback(logdir="./logs", loader_key="valid", metric_key="loss", minimize=True, save_n_best=3),
 ]
 # model training
 runner.train(
@@ -843,16 +827,19 @@ runner.train(
 import os
 from torch.optim import Adam
 from torch.utils.data import DataLoader
-from catalyst import data, dl
-from catalyst.contrib import datasets, models, nn
-from catalyst.data.transforms import Compose, Normalize, ToTensor
+from catalyst import dl
+from catalyst.data import BatchBalanceClassSampler
+from catalyst.contrib import data, datasets, models, nn
 
 
 # 1. train and valid loaders
-transforms = Compose([ToTensor(), Normalize((0.1307,), (0.3081,))])
+transforms = data.Compose([
+    data.ImageToTensor(), 
+    data.NormalizeImage((0.1307,), (0.3081,))
+])
 
 train_dataset = datasets.MnistMLDataset(root=os.getcwd(), download=True, transform=transforms)
-sampler = data.BatchBalanceClassSampler(
+sampler = BatchBalanceClassSampler(
     labels=train_dataset.get_labels(), num_classes=5, num_samples=10, num_batches=10
 )
 train_loader = DataLoader(dataset=train_dataset, batch_sampler=sampler)
@@ -929,9 +916,7 @@ import torch
 from torch import nn
 from torch.utils.data import DataLoader
 from catalyst import dl
-from catalyst.contrib.datasets import MNIST
-from catalyst.contrib.nn.modules import Flatten, GlobalMaxPool2d, Lambda
-from catalyst.data import ToTensor
+from catalyst.contrib import Flatten, GlobalMaxPool2d, Lambda, MNIST, ImageToTensor
 
 latent_dim = 128
 generator = nn.Sequential(
@@ -962,7 +947,7 @@ optimizer = {
     "generator": torch.optim.Adam(generator.parameters(), lr=0.0003, betas=(0.5, 0.999)),
     "discriminator": torch.optim.Adam(discriminator.parameters(), lr=0.0003, betas=(0.5, 0.999)),
 }
-train_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+train_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
 loaders = {"train": DataLoader(train_data, batch_size=32)}
 
 class CustomRunner(dl.Runner):
@@ -1070,8 +1055,7 @@ from torch import nn, optim
 from torch.nn import functional as F
 from torch.utils.data import DataLoader
 from catalyst import dl, metrics
-from catalyst.contrib.datasets import MNIST
-from catalyst.data import ToTensor
+from catalyst.contrib import ImageToTensor, MNIST
 
 LOG_SCALE_MAX = 2
 LOG_SCALE_MIN = -10
@@ -1124,8 +1108,8 @@ class CustomRunner(dl.IRunner):
         return 3
 
     def get_loaders(self, stage: str):
-        train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-        valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+        train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+        valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
         loaders = {
             "train": DataLoader(train_data, batch_size=32),
             "valid": DataLoader(valid_data, batch_size=32),
@@ -1197,8 +1181,7 @@ import os
 from torch import nn, optim
 from torch.utils.data import DataLoader
 from catalyst import dl, utils
-from catalyst.contrib.datasets import MNIST
-from catalyst.data import ToTensor
+from catalyst.contrib import ImageToTensor, MNIST
 
 
 class CustomRunner(dl.IRunner):
@@ -1225,8 +1208,8 @@ class CustomRunner(dl.IRunner):
         return 3
 
     def get_loaders(self, stage: str):
-        train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-        valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+        train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+        valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
         loaders = {
             "train": DataLoader(train_data, batch_size=32),
             "valid": DataLoader(valid_data, batch_size=32),
@@ -1305,8 +1288,7 @@ import os
 from torch import nn, optim
 from torch.utils.data import DataLoader
 from catalyst import dl, utils
-from catalyst.contrib.datasets import MNIST
-from catalyst.data import ToTensor
+from catalyst.contrib import ImageToTensor, MNIST
 
 
 class CustomRunner(dl.IRunner):
@@ -1332,8 +1314,8 @@ class CustomRunner(dl.IRunner):
         return 3
 
     def get_loaders(self, stage: str):
-        train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-        valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+        train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+        valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
         loaders = {
             "train": DataLoader(train_data, batch_size=32),
             "valid": DataLoader(valid_data, batch_size=32),
@@ -1420,16 +1402,15 @@ import torch
 from torch import nn
 from torch.utils.data import DataLoader
 from catalyst import dl
-from catalyst.data import ToTensor
-from catalyst.contrib.datasets import MNIST
+from catalyst.contrib import ImageToTensor, MNIST
 
 
 def objective(trial):
     lr = trial.suggest_loguniform("lr", 1e-3, 1e-1)
     num_hidden = int(trial.suggest_loguniform("num_hidden", 32, 128))
 
-    train_data = MNIST(os.getcwd(), train=True, download=True, transform=ToTensor())
-    valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ToTensor())
+    train_data = MNIST(os.getcwd(), train=True, download=True, transform=ImageToTensor())
+    valid_data = MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor())
     loaders = {
         "train": DataLoader(train_data, batch_size=32),
         "valid": DataLoader(valid_data, batch_size=32),