Navigation filter (#12)

DariuszPorowski · web-flow · commit 046e04247996 · 2023-02-08T18:37:19.000-08:00
close #9
diff --git a/.editorconfig b/.editorconfig
@@ -10,7 +10,7 @@ indent_style = tab
 [*.{cmd,bat}]
 end_of_line = crlf
 
-[*.{yml,yaml,md}]
+[*.{yml,yaml,md,pages}]
 indent_style = space
 indent_size = 2
 
diff --git a/.file-filter.yml b/.file-filter.yml
@@ -1,11 +1,11 @@
-enabled: !ENV [CI, true]
 mkdocsignore: false
 include_glob:
   - 'test/test.md'
   - '**/draft-*.md'
+exclude_glob:
+  - 'Contributions/**'
 include_tag:
   - foo
   - '#bar'
-enabled_on_serve: !ENV [CI, true]
 exclude_tag:
   - test
diff --git a/.github/workflows/workflow.pr.yml b/.github/workflows/workflow.pr.yml
@@ -45,7 +45,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "pypy-3.8"]
+        python-version: ["3.8", "3.9", "3.10"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -67,4 +67,10 @@ jobs:
       - name: Test Build
         run: |
           poetry version
-          poetry build
+          poetry build --verbose
+
+      - name: Test MkDocs build
+        run: |
+          pip uninstall -y mkdocs_file_filter_plugin
+          pip install dist/mkdocs_file_filter_plugin-0.0.1.tar.gz
+          poetry run mkdocs build --verbose
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 # MkDocs
 site/**
+docs/Contributions
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -11,3 +11,7 @@ repos:
     rev: 6.0.0
     hooks:
     - id: flake8
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.7.4
+    hooks:
+    - id: bandit
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -4,12 +4,8 @@
 			"mkdocs.yml",
 			"mkdocs.base.yml"
 		],
-		// "https://json.schemastore.org/mkdocs-1.0.json": [
-		// 	"mkdocs.yml",
-		// 	"mkdocs.base.yml"
-		// ],
-		"schema-config.json": [
-			"mkdocs.file-filter.yml"
+		"schema-file-filter-config.json": [
+			".file-filter.yml"
 		]
 	},
 	"files.trimTrailingWhitespace": true,
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# mkdocs-file-filter-plugin
+# MkDocs File Filter Plugin
 
 [![PyPI - Version][pypi-v-image]][pypi-v-link]
 [![PyPI - Python Version][pypi-pyversions-image]][pypi-pyversions-link]
@@ -23,6 +23,7 @@
     - [Tags metadata property](#tags-metadata-property)
     - [Custom metadata list](#custom-metadata-list)
   - [.mkdocsignore](#mkdocsignore)
+  - [Navigation filtering](#navigation-filtering)
   - [Conflict behavior](#conflict-behavior)
 - [License](#license)
 
@@ -70,8 +71,9 @@ Add a plugin configuration to `mkdocs.yml` - below example contains only example
 plugins:
   - search # if you include another plugin, and want search you have to add it again
   - file-filter:
+      filter_nav: true # default value
       exclude_glob:
-        - 'exclude/this/path/*'
+        - 'exclude/this/path/**'
         - 'exclude/this/file/draft.md'
         - '*.tmp'
       exclude_regex:
@@ -80,7 +82,7 @@ plugins:
         - draft
         - preview
       include_glob:
-        - 'include/this/path/*'
+        - 'include/this/path/**'
         - 'include/this/file/Code-of-Conduct.md'
         - '*.png'
         - 'assets/**' # the material theme requires this folder
@@ -105,7 +107,8 @@ The plugin supports external files for the plugin configuration. If the external
 plugins:
   - search # if you include another plugin, and want search you have to add it again
   - file-filter:
-      config: !ENV [MY_FILE_FILTER_CONFIG, 'mkdocs.file-filter.yml']
+      config: !ENV [MY_FILE_FILTER_CONFIG, '.file-filter.yml']
+      # config: path/to/config/file/.file-filter.yml
 ```
 
 > :bulb: **HINT**
@@ -118,6 +121,7 @@ External plugin config file example:
 # mkdocs.file-filter.yml
 enabled: !ENV [CI, true]
 enabled_on_serve: true
+filter_nav: true
 mkdocsignore: false
 exclude_glob:
   - 'exclude/this/path/*'
@@ -153,9 +157,10 @@ The below table shows all supported options by the plugin.
 | `config` | string | *none* | Path to external plugin's configuration file |
 | `enabled` | bool | `true` | Turn on/off plugin without removing/adding plugin's config from `mkdocs.yml` |
 | `enabled_on_serve` | bool | `false` | Turn on/off plugin on `serve` command |
+| `filter_nav` | bool | `true` | Remove `nav` items pointed to excluded files |
 | `mkdocsignore` | bool | `false` | Use gitignore-style file for patterns |
 | `mkdocsignore_file` | string | `.mkdocsignore` | Path to gitignore-style file with patterns |
-| `metadata_property` | string | `tags` | What Markdown/FrontMatter metadata property will be used for checking tags |
+| `metadata_property` | string | `tags` | What Markdown/FrontMatter metadata property list will be used for checking keywords |
 | `exclude_tag` | [string] | *none* | List of excluded tags |
 | `include_tag` | [string] | *none* | List of included tags |
 | `exclude_glob` | [string] | *none* | Exclude glob patterns |
@@ -181,6 +186,7 @@ The below table shows all supported options by the plugin.
 # mkdocs.yml
 plugins:
   - file-filter:
+      filter_nav: true # default value
       exclude_glob:
         - 'drafts/**'
       include_glob:
@@ -199,6 +205,7 @@ plugins:
 # mkdocs.yml
 plugins:
   - file-filter:
+      filter_nav: true # default value
       exclude_regex:
         - '.*\.(tmp|bin|tar)$'
       include_regex:
@@ -223,6 +230,7 @@ By default plugin filter files using `tags` property of your Markdown metadata.
 # mkdocs.yml
 plugins:
   - file-filter:
+      filter_nav: true # default value
       exclude_tag:
         - abc
         - draft
@@ -275,6 +283,7 @@ Set `metadata_property` with your custom list property, e.g., `labels` for this
 # mkdocs.yml
 plugins:
   - file-filter:
+      filter_nav: true # default value
       metadata_property: labels
       exclude_tag:
         - foo
@@ -336,8 +345,9 @@ Example config for mkdocsignore.
 # mkdocs.yml
 plugins:
   - file-filter:
+      filter_nav: true # default value
       mkdocsignore: true # default: false
-      mkdocsignore_file: 'custom/path/.myignore' # optional, relative to mkdocs.yml, default: .mkdocsignore
+      mkdocsignore_file: 'custom/path/.mkdocsignore' # optional, relative to mkdocs.yml, default: .mkdocsignore
 ```
 
 Example `.mkdocsignore` file.
@@ -354,6 +364,59 @@ docs/**/draft-*.md
 >
 > **.mkdocsignore** patterns relative to your root.
 
+### Navigation filtering
+
+Suppose you customized [MkDocs navigation configuration][mkdocs-nav], and your `nav` contains elements defined in exclude patterns. In that case, the default MkDocs behavior is to render navigation to a non-existing file, and generated site gives 404.
+
+By default, the plugin filters those cases and removes not working navigation items.
+
+You can control the plugin's behavior to explicitly disable that option by setting `filter_nav: false`.
+
+Example `mkdocs.yml` config.
+
+```yaml
+# mkdocs.yml
+nav:
+- Foo: exclude/this/path
+- Bar: exclude/this/file/draft.md
+- Abc:
+    - About: exclude/this/path/about.md
+    - Contact: include/this/file/contact.md
+- Xyz: path/xyz.md
+
+plugins:
+  - file-filter:
+      filter_nav: true # default value
+      exclude_glob:
+        - 'exclude/this/path/**'
+        - 'exclude/this/file/draft.md'
+```
+
+**Nav** results with `filter_nav: false`:
+
+```yaml
+- Foo: exclude/this/path # -> 404
+- Bar: exclude/this/file/draft.md # -> 404
+- Abc:
+    - About: exclude/this/path/about.md # -> 404
+    - Contact: include/this/file/contact.md
+- Xyz: path/xyz.md
+```
+
+**Nav** results with `filter_nav: true`:
+
+```yaml
+- Abc:
+    - Contact: include/this/file/contact.md
+- Xyz: path/xyz.md
+```
+
+> :warning: **NOTE**
+>
+> If you use any other 3rd-party plugins that modify navigation (e.g., [mkdocs-awesome-pages-plugin][mkdocs-awesome-pages-plugin-pypi]) - first, test and evaluate expected behavior. Plugin configuration order may impact results.
+
+> In most cases is recommended to transform navigation first and next filter with the file-filter plugin.
+
 ### Conflict behavior
 
 It is possible to exclude and include will have conflict. For example, you could exclude `drafts/**` but include `*.md`. In that case, **include** has higher priority over exclude. So all `*.md` files from the drafts folder will be **included**.
@@ -370,7 +433,9 @@ It is possible to exclude and include will have conflict. For example, you could
 [mkdocs-envs]: https://www.mkdocs.org/user-guide/configuration/#environment-variables
 [mkdocs-metadata]: https://www.mkdocs.org/user-guide/writing-your-docs/#meta-data
 [mkdocs-docs-dir]: https://www.mkdocs.org/user-guide/configuration/#docs_dir
+[mkdocs-nav]: https://www.mkdocs.org/user-guide/writing-your-docs/#configure-pages-and-navigation
 [poetry]: https://python-poetry.org
 [pip]: https://pip.pypa.io
 [gitignore]: https://git-scm.com/docs/gitignore
-[mit]: https://spdx.org/licenses/MIT.html
+[mit]: https://opensource.org/licenses/MIT
+[mkdocs-awesome-pages-plugin-pypi]: https://pypi.org/project/mkdocs-awesome-pages-plugin
diff --git a/dev.build.sh b/dev.build.sh
@@ -5,6 +5,7 @@ rm -rf dist && rm -rf build && rm -rf site && rm -rf *.egg-info
 poetry run isort .
 poetry run black .
 poetry run flake8 --count .
+# poetry run bandit --recursive .
 
 poetry build
 
diff --git a/docs/NAV.md b/docs/NAV.md
@@ -2,8 +2,8 @@
 title: nav
 ---
 
-* [Frob](index.md)
+<!-- * [Frob](index.md)
 * [Baz](baz.md)
 * [Borgs](borgs/index.md)
   * [Bar](borgs/bar.md)
-  * [Foo](borgs/foo.md)
+  * [Foo](borgs/foo.md) -->
diff --git a/mkdocs.base.yml b/mkdocs.base.yml
@@ -2,10 +2,3 @@ site_name: Test Site
 
 theme:
   name: material
-
-plugins:
-  - monorepo
-  - awesome-pages
-  # - literate-nav:
-  #     nav_file: NAV.md
-
diff --git a/mkdocs.yml b/mkdocs.yml
@@ -4,10 +4,12 @@ INHERIT: mkdocs.base.yml
 
 plugins:
   - search
+  # - monorepo
   - tags:
       tags_file: tags.md
+  - awesome-pages
   - file-filter:
-      config: !ENV [MY_FILE_FILTER_CONFIG, "mkdocs.file-filter.yml"]
-      # metadata_property: labels
-      # exclude_tag:
-      #   - test
+      config: !ENV [MY_FILE_FILTER_CONFIG, ".file-filter.yml"]
+      enabled: !ENV [CI, true]
+      enabled_on_serve: true
+      filter_nav: true
diff --git a/mkdocs_file_filter_plugin/external_config.py b/mkdocs_file_filter_plugin/external_config.py
@@ -24,6 +24,7 @@ def __init__(self):
                 Optional("include_glob"): [str],
                 Optional("include_regex"): [str],
                 Optional("include_tag"): [str],
+                Optional("filter_nav"): bool,
             }
         )
 
diff --git a/mkdocs_file_filter_plugin/judger.py b/mkdocs_file_filter_plugin/judger.py
@@ -2,12 +2,16 @@
 import os
 import pathlib
 import re
+from urllib.parse import urlsplit
 
 import igittigitt
 from mkdocs.config.defaults import MkDocsConfig
 from mkdocs.structure.files import File as MkDocsFile
+from mkdocs.structure.nav import Link as MkDocsLink
+from mkdocs.structure.nav import Section as MkDocsSection
 from mkdocs.structure.pages import Page as MkDocsPage
 
+from . import util as LOG
 from .plugin_config import PluginConfig
 
 
@@ -22,44 +26,62 @@ def __init__(self, plugin_config: PluginConfig, mkdocs_config: MkDocsConfig):
                 pathlib.Path(self.plugin_config.mkdocsignore_file)
             )
 
-    def evaluate(self, file: MkDocsFile):
+    def evaluate_nav(self, nav):
+        if isinstance(nav, MkDocsSection):
+            nev_section = [self.evaluate_nav(child) for child in nav.children]
+            nev_section = list(filter(lambda item: item is not None, nev_section))
+            if nev_section != []:
+                return MkDocsSection(nav.title, nev_section)
+            else:
+                LOG.debug(f"remove navigation section: {nav.title}")
+        else:
+            scheme, netloc, path, query, fragment = urlsplit(nav.url)
+            if (
+                isinstance(nav, MkDocsLink)
+                and not nav.url.startswith("/")
+                and not scheme
+                and not netloc
+            ):
+                LOG.debug(f"remove navigation item: {nav.title} {nav.url}")
+            else:
+                return nav
+
+    def evaluate_file(self, file: MkDocsFile):
         file.src_path, file.abs_src_path = self.__path_fix(
             file.src_path, file.abs_src_path
         )
         for glob in self.plugin_config.include_glob:
             if fnmatch.fnmatchcase(file.src_path, glob):
-                return file, True, str(f"glob: {glob}")
+                return True, str(f"glob: {glob}")
         for regex in self.plugin_config.include_regex:
             if re.match(regex, file.src_path):
-                return file, True, str(f"regex: {regex}")
+                return True, str(f"regex: {regex}")
         if file.is_documentation_page() and self.plugin_config.include_tag is not []:
             tags = self.__get_metadata(file)
             for tag in self.plugin_config.include_tag:
                 if tag in tags:
                     return (
-                        file,
                         True,
                         str(f"{self.plugin_config.metadata_property}: {tag}"),
                     )
         for glob in self.plugin_config.exclude_glob:
             if fnmatch.fnmatchcase(file.src_path, glob):
-                return file, False, str(f"glob: {glob}")
+                return False, str(f"glob: {glob}")
         for regex in self.plugin_config.exclude_regex:
             if re.match(regex, file.src_path):
-                return file, False, str(f"regex: {regex}")
+                return False, str(f"regex: {regex}")
         if file.is_documentation_page() and self.plugin_config.exclude_tag is not []:
             tags = self.__get_metadata(file)
             for tag in self.plugin_config.exclude_tag:
                 if tag in tags:
                     return (
-                        file,
                         False,
                         str(f"{self.plugin_config.metadata_property}: {tag}"),
                     )
         if self.plugin_config.mkdocsignore is True:
             if self.mkdocsignore_parser.match(pathlib.Path(file.abs_src_path)):
                 return file, False, "mkdocsignore"
-        return file, True, "no rule"
+        return True, "no rule"
 
     def __path_fix(self, src_path, abs_src_path):
         if os.sep != "/":
diff --git a/mkdocs_file_filter_plugin/plugin.py b/mkdocs_file_filter_plugin/plugin.py
diff --git a/mkdocs_file_filter_plugin/plugin_config.py b/mkdocs_file_filter_plugin/plugin_config.py
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/schema-file-filter-config.json b/schema-file-filter-config.json
diff --git a/schema.json b/schema.json
