Support character encoding setting

docs/problemset_config.rst

@@ -16,6 +16,13 @@ problemset.toml の書き方
 
 設定項目それぞれについて説明します。
 
+.. problemsettoml:: encoding
+
+    入力ファイルおよび出力ファイルの文字コードを指定します。`Python の codec 文字列 <https://docs.python.org/ja/3/library/codecs.html#standard-encodings>`_ を指定してください。
+
+    何も設定しなかった場合、 ``utf-8`` が適用されます。
+
+
 .. problemsettoml:: [template]
 
     .. problemsettoml:: template_path

statements_manager/src/convert_task_runner.py

@@ -41,7 +41,8 @@ def __init__(self, problemset_config: ProblemSetConfig):
     def get_local_contents(self, problem_id: str) -> Tuple[ContentsStatus, str]:
         try:
             with open(
-                self.problemset_config.get_problem(problem_id).statement.path
+                self.problemset_config.get_problem(problem_id).statement.path,
+                encoding=self.problemset_config.encoding,
             ) as f:
                 return (ContentsStatus.OK, f.read())
         except EnvironmentError:
@@ -104,7 +105,7 @@ def get_contents(self, problem_id: str) -> Tuple[ContentsStatus, str]:
             raise ValueError(f"unknown mode: {mode}")
 
     def save_file(self, text: str, output_path: str):
-        with open(output_path, "w") as f:
+        with open(output_path, "w", encoding=self.problemset_config.encoding) as f:
             f.write(text)
 
     # 制約パラメータファイル (e.g. constraints.hpp) の作成
@@ -118,8 +119,9 @@ def create_params_file(self, problem_id: str) -> None:
             ext: str = Path(problem_config.params_path).suffix
             if ext in lang_to_class:
                 params_maker = lang_to_class[ext](
-                    problem_config.constraints,
-                    problem_config.params_path,
+                    params=problem_config.constraints,
+                    output_path=problem_config.params_path,
+                    encoding=self.problemset_config.encoding,
                 )
                 params_maker.run()
             else:

statements_manager/src/execute_config.py

@@ -183,14 +183,20 @@ def __init__(self, problemset_filename: pathlib.Path, config: dict) -> None:
         self.known_ids: set[str] = set()
         self.id_groups: list[list[str]] = list()
         self.problem_configs: dict[str, ProblemConfig] = dict()
+        self.encoding: str = self.optional(
+            problemset_filename, config, "encoding", "utf-8"
+        )
+        logger.info(f"encoding = {self.encoding}")
 
         dirname = problemset_filename.parent.resolve()
         self.output_path = dirname / "problemset"
         self.template_html: str = read_text_file(
-            to_path(self.template.template_path), default_template_html
+            to_path(self.template.template_path), default_template_html, self.encoding
         )
         self.sample_template_html: str = read_text_file(
-            to_path(self.template.sample_template_path), default_sample_template_html
+            to_path(self.template.sample_template_path),
+            default_sample_template_html,
+            self.encoding,
         )
 
     def get_problem(self, id: str) -> ProblemConfig:

statements_manager/src/params_maker/languages/cplusplus.py

@@ -6,8 +6,8 @@
 
 
 class CppParamsMaker(ParamsMaker):
-    def __init__(self, params: dict[str, Any], output_path: str) -> None:
-        super().__init__(params, output_path)
+    def __init__(self, params: dict[str, Any], output_path: str, encoding: str) -> None:
+        super().__init__(params, output_path, encoding)
 
     def header(self) -> str:
         return """// DO NOT EDIT THIS FILE MANUALLY

statements_manager/src/params_maker/params_maker.py

@@ -9,9 +9,10 @@
 
 
 class ParamsMaker:
-    def __init__(self, params: dict[str, Any], output_path: str) -> None:
+    def __init__(self, params: dict[str, Any], output_path: str, encoding: str) -> None:
         self.params = params
         self.output_path = output_path
+        self.encoding = encoding
 
     def run(self) -> None:
         params_lines: list[str] = []
@@ -35,15 +36,15 @@ def run(self) -> None:
         # if params file is the same as the existing one, do nothing
         params_text = "\n".join(params_lines)
         if pathlib.Path(self.output_path).exists():
-            with open(self.output_path, "r") as f:
+            with open(self.output_path, "r", encoding=self.encoding) as f:
                 reference = f.read()
                 if params_text == reference:
                     logger.warning(
                         "skip dumping constraints file: same result as before"
                     )
                     return
 
-        with open(self.output_path, "w") as f:
+        with open(self.output_path, "w", encoding=self.encoding) as f:
             f.write(params_text)
 
     @abstractmethod

statements_manager/src/render_result_cache.py

@@ -37,7 +37,7 @@ def __init__(
     def _load_cache(self) -> dict:
         if not self.cache_path.exists():
             return {}
-        with open(self.cache_path, "r") as f:
+        with open(self.cache_path, "r", encoding="ascii") as f:
             return self._cleanup(json.load(f))
 
     def _setup_cache(self, cache: dict) -> dict:
@@ -101,7 +101,7 @@ def need_to_save(self, force_dump: bool):
     def save_and_check_diff(self) -> bool:
         json.dump(
             self.cache,
-            open(self.cache_path, "w"),
+            open(self.cache_path, "w", encoding="ascii"),
             indent=4,
             sort_keys=True,
         )

statements_manager/src/renderer.py

@@ -59,12 +59,14 @@ def __init__(
         self.replace_sample_format = ReplaceSampleFormatExprExtension()
 
     def replace_vars(
-        self, problem_config: ProblemConfig, statement_str: str | None
+        self, problem_config: ProblemConfig, statement_str: str | None, encoding: str
     ) -> str:
         if statement_str is None:
             logger.error("statement_str is None")
             raise RuntimeError("statement_str is None")
-        vars_manager = VariablesConverter(problem_config, self.sample_template_html)
+        vars_manager = VariablesConverter(
+            problem_config, self.sample_template_html, encoding
+        )
         env = Environment(
             variable_start_string="{@",
             variable_end_string="}",
@@ -188,7 +190,9 @@ def generate_html(
                 contents = problem_config.statement.raw_text
                 contents = self.apply_preprocess(contents)
 
-                rendered_contents = self.replace_vars(problem_config, contents)
+                rendered_contents = self.replace_vars(
+                    problem_config, contents, problemset_config.encoding
+                )
                 markdown_extensions = [
                     self.replace_sample_format,
                     *problem_config.statement.markdown_extensions,
@@ -246,7 +250,9 @@ def generate_markdown(
             problem_config = problemset_config.get_problem(problem_id)
             if problem_config.statement.rendered_text is None:
                 contents = problem_config.statement.raw_text
-                contents = self.replace_vars(problem_config, contents)
+                contents = self.replace_vars(
+                    problem_config, contents, problemset_config.encoding
+                )
                 problem_config.statement.rendered_text = contents
             problem_config.statement.rendered_text = self.replace_assets_path(
                 problem_config.statement.rendered_text, problem_id, is_problemset

statements_manager/src/utils.py

@@ -30,10 +30,10 @@ def read_toml_file(path: Path | None) -> dict:
     return toml.load(path)
 
 
-def read_text_file(path: Path | None, default: str) -> str:
+def read_text_file(path: Path | None, default: str, encoding: str) -> str:
     if path is None or not path.exists():
         return default
-    with path.open() as f:
+    with path.open("r", encoding=encoding) as f:
         return f.read()
 
 

statements_manager/src/variables_converter.py

@@ -40,9 +40,9 @@ def to_string(value: Any, config: StatementConfig) -> str:
         return str(value)
 
 
-def fetch_text(path: pathlib.Path) -> str:
+def fetch_text(path: pathlib.Path, encoding: str) -> str:
     try:
-        return open(path).read()
+        return open(path, encoding=encoding).read()
     except OSError:
         return ""
 
@@ -63,22 +63,6 @@ def convert(
             logger.warning("constraints are not set")
 
 
-class SampleFile:
-    def __init__(self, filename: pathlib.Path) -> None:
-        self.filename = filename
-
-    def exists(self) -> bool:
-        return self.filename.exists()
-
-    def print_raw(self) -> str:
-        if not self.exists():
-            return ""
-
-        with open(self.filename, "r") as f:
-            contents = f.read() + "\n"
-            return contents
-
-
 class SampleData:
     def __init__(
         self,
@@ -176,7 +160,11 @@ def print_warning(
             logger.warning(f"{sample_name}: There is no explanation.")
 
     def convert(
-        self, samples: dict[str, Any], problem_config: ProblemConfig, template: str
+        self,
+        samples: dict[str, Any],
+        problem_config: ProblemConfig,
+        template: str,
+        encoding: str,
     ) -> None:
         """
         - `sample_path` が指定されていない場合: 警告を出して抜ける
@@ -227,13 +215,13 @@ def convert(
             )
 
             if input_file.exists():
-                sample_data.input_text = fetch_text(input_file)
+                sample_data.input_text = fetch_text(input_file, encoding)
             if output_file.exists():
-                sample_data.output_text = fetch_text(output_file)
+                sample_data.output_text = fetch_text(output_file, encoding)
             if md_file.exists():
-                sample_data.md_text = fetch_text(md_file)
+                sample_data.md_text = fetch_text(md_file, encoding)
             if explanation_file.exists():
-                sample_data.explanation_text = fetch_text(explanation_file)
+                sample_data.explanation_text = fetch_text(explanation_file, encoding)
 
             sample_text = env.get_template("template").render(
                 sample_data=sample_data.to_dict(),
@@ -245,11 +233,15 @@ def convert(
 
 
 class VariablesConverter:
-    def __init__(self, problem_config: ProblemConfig, sample_template: str) -> None:
+    def __init__(
+        self, problem_config: ProblemConfig, sample_template: str, encoding: str
+    ) -> None:
         self.constraints: dict = {}
         self.samples: dict = {}
         self.constraints_converter = ConstraintsConverter()
         self.samples_converter = SamplesConverter()
 
         self.constraints_converter.convert(self.constraints, problem_config)
-        self.samples_converter.convert(self.samples, problem_config, sample_template)
+        self.samples_converter.convert(
+            self.samples, problem_config, sample_template, encoding
+        )