feat: Investigate and fix issue with wrong CPU count for containers (#623)

So, I found that `nproc` always shows how many CPUs available is. K8s "limits" and docker `--cpus` are throttling mechanisms, which do not hide the visibility of all cores.
There are a few workarounds, but  IMO, it is better to implement checks for that than do them

>Workaround for docker - set `--cpuset-cpus`
>Workaraund for K8s - somehow deal with [kubelet static CPU management policy](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/#cpu-management-policies), as [recommend in Reddit](https://news.ycombinator.com/item?id=25224714)

* Send all "colorify" logs through stderr, as make able to add user-facing-logs in functions that also need to return same value to the function-caller. Needed for `common::get_cpu_num` err_msg show up

---------

Co-authored-by: George L. Yermulnik <[email protected]>

Author: MaxymVlasov

README.md

@@ -393,6 +393,16 @@ If you'd like to set parallelism value relative to number of CPU logical cores -
 >
 > </details>
 
+
+
+```yaml
+args:
+  - --hook-config=--parallelism-cpu-cores=N
+```
+
+If you don't see code above in your `pre-commit-config.yaml` or logs - you don't need it.  
+`--parallelism-cpu-cores` used only in edge cases. Check-out it usage in [hooks/_common.sh](hooks/_common.sh)
+
 ### checkov (deprecated) and terraform_checkov
 
 > `checkov` hook is deprecated, please use `terraform_checkov`.

hooks/_common.sh

@@ -170,6 +170,68 @@ function common::is_hook_run_on_whole_repo {
   fi
 }
 
+#######################################################################
+# Get the number of CPU logical cores available for pre-commit to use
+# Arguments:
+#  parallelism_cpu_cores (string) Used in edge cases when number of
+#    CPU cores can't be derived automatically
+# Outputs:
+#   Returns number of CPU logical cores, rounded down to nearest integer
+#######################################################################
+function common::get_cpu_num {
+  local -r parallelism_cpu_cores=$1
+
+  if [[ -n $parallelism_cpu_cores ]]; then
+    # 22 EINVAL Invalid argument. Some invalid argument was supplied.
+    [[ $parallelism_cpu_cores =~ ^[[:digit:]]+$ ]] || return 22
+
+    echo "$parallelism_cpu_cores"
+    return
+  fi
+
+  local millicpu
+
+  if [[ -f /sys/fs/cgroup/cpu/cpu.cfs_quota_us ]]; then
+    # Inside K8s pod or DInD in K8s
+    millicpu=$(< /sys/fs/cgroup/cpu/cpu.cfs_quota_us)
+
+    if [[ $millicpu -eq -1 ]]; then
+      # K8s no limits or in DinD
+      common::colorify "yellow" "Unable to derive number of available CPU cores.\n" \
+        "Running inside K8s pod without limits or inside DinD without limits propagation.\n" \
+        "To avoid possible harm, parallelism is disabled.\n" \
+        "To re-enable it, set corresponding limits, or set the following for the current hook:\n" \
+        "  args:\n" \
+        "    - --hook-config=--parallelism-ci-cpu-cores=N\n" \
+        "where N is the number of CPU cores to allocate to pre-commit."
+
+      echo 1
+      return
+    fi
+
+    echo $((millicpu / 1000))
+    return
+  fi
+
+  if [[ -f /sys/fs/cgroup/cpu.max ]]; then
+    # Inside Linux (Docker?) container
+    millicpu=$(cut -d' ' -f1 /sys/fs/cgroup/cpu.max)
+
+    if [[ $millicpu == max ]]; then
+      # No limits
+      nproc 2> /dev/null || echo 1
+      return
+    fi
+
+    echo $((millicpu / 1000))
+    return
+  fi
+
+  # On host machine or any other case
+  # `nproc` - linux, `sysctl -n hw.ncpu` - macOS, `echo 1` - fallback
+  nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 1
+}
+
 #######################################################################
 # Hook execution boilerplate logic which is common to hooks, that run
 # on per dir basis.
@@ -219,13 +281,8 @@ function common::per_dir_hook {
 
   # Lookup hook-config for modifiers that impact common behavior
   local change_dir_in_unique_part=false
-  # Limit the number of parallel processes to the number of CPU cores -1
-  # `nproc` - linux, `sysctl -n hw.ncpu` - macOS, `echo 1` - fallback
-  local CPU
-  CPU=$(nproc 2> /dev/null || sysctl -n hw.ncpu 2> /dev/null || echo 1)
-  local parallelism_limit
-  local parallelism_disabled=false
 
+  local parallelism_limit
   IFS=";" read -r -a configs <<< "${HOOK_CONFIG[*]}"
   for c in "${configs[@]}"; do
     IFS="=" read -r -a config <<< "$c"
@@ -246,11 +303,21 @@ function common::per_dir_hook {
         ;;
       --parallelism-limit)
         # this flag will limit the number of parallel processes
-        parallelism_limit=$((value))
+        parallelism_limit="$value"
+        ;;
+      --parallelism-cpu-cores)
+        # Used in edge cases when number of CPU cores can't be derived automatically
+        parallelism_cpu_cores="$value"
         ;;
     esac
   done
 
+  CPU=$(common::get_cpu_num "$parallelism_cpu_cores")
+
+  # parallelism_limit can include reference to 'CPU' variable
+  parallelism_limit=$((parallelism_limit))
+  local parallelism_disabled=false
+
   if [[ ! $parallelism_limit ]]; then
     parallelism_limit=$((CPU - 1))
   elif [[ $parallelism_limit -le 1 ]]; then
@@ -321,14 +388,15 @@ function common::colorify {
 
   # Params start #
   local COLOR="${!1}"
-  local -r TEXT=$2
+  shift
+  local -r TEXT="$*"
   # Params end #
 
   if [ "$PRE_COMMIT_COLOR" = "never" ]; then
     COLOR=$RESET
   fi
 
-  echo -e "${COLOR}${TEXT}${RESET}"
+  echo -e "${COLOR}${TEXT}${RESET}" >&2
 }
 
 #######################################################################