distill, finetune: set the default number of warmup steps to 10,000

danieldk · Daniël de Kok · commit 99b13e08ee77 · 2021-03-19T10:48:01.000+01:00
This has empirically proven to be a better default than 2000.
diff --git a/syntaxdot-cli/src/subcommands/distill.rs b/syntaxdot-cli/src/subcommands/distill.rs
@@ -1118,7 +1118,7 @@ impl SyntaxDotApp for DistillApp {
                     .help(
                         "For the first N timesteps, the learning rate is linearly scaled up to LR.",
                     )
-                    .default_value("2000"),
+                    .default_value("10000"),
             )
             .arg(
                 Arg::with_name(WEIGHT_DECAY)
diff --git a/syntaxdot-cli/src/subcommands/finetune.rs b/syntaxdot-cli/src/subcommands/finetune.rs
@@ -536,7 +536,7 @@ impl SyntaxDotApp for FinetuneApp {
                     .help(
                         "For the first N timesteps, the learning rate is linearly scaled up to LR.",
                     )
-                    .default_value("2000"),
+                    .default_value("10000"),
             )
             .arg(
                 Arg::with_name(WEIGHT_DECAY)

Original file line number	Diff line number	Diff line change
`@@ -1118,7 +1118,7 @@ impl SyntaxDotApp for DistillApp {`
`1118`	`1118`	`.help(`
`1119`	`1119`	`"For the first N timesteps, the learning rate is linearly scaled up to LR.",`
`1120`	`1120`	`)`
`1121`		`- .default_value("2000"),`
	`1121`	`+ .default_value("10000"),`
`1122`	`1122`	`)`
`1123`	`1123`	`.arg(`
`1124`	`1124`	`Arg::with_name(WEIGHT_DECAY)`
Original file line number	Diff line number	Diff line change
`@@ -536,7 +536,7 @@ impl SyntaxDotApp for FinetuneApp {`
`536`	`536`	`.help(`
`537`	`537`	`"For the first N timesteps, the learning rate is linearly scaled up to LR.",`
`538`	`538`	`)`
`539`		`- .default_value("2000"),`
	`539`	`+ .default_value("10000"),`
`540`	`540`	`)`
`541`	`541`	`.arg(`
`542`	`542`	`Arg::with_name(WEIGHT_DECAY)`