diff --git a/docs/problemset_config.rst b/docs/problemset_config.rst index 52680a3..ff830d7 100644 --- a/docs/problemset_config.rst +++ b/docs/problemset_config.rst @@ -16,6 +16,13 @@ problemset.toml の書き方 設定項目それぞれについて説明します。 +.. problemsettoml:: encoding + + 入力ファイルおよび出力ファイルの文字コードを指定します。`Python の codec 文字列 `_ を指定してください。 + + 何も設定しなかった場合、 ``utf-8`` が適用されます。 + + .. problemsettoml:: [template] .. problemsettoml:: template_path diff --git a/statements_manager/src/convert_task_runner.py b/statements_manager/src/convert_task_runner.py index 71cd7a8..b516ef4 100644 --- a/statements_manager/src/convert_task_runner.py +++ b/statements_manager/src/convert_task_runner.py @@ -41,7 +41,8 @@ def __init__(self, problemset_config: ProblemSetConfig): def get_local_contents(self, problem_id: str) -> Tuple[ContentsStatus, str]: try: with open( - self.problemset_config.get_problem(problem_id).statement.path + self.problemset_config.get_problem(problem_id).statement.path, + encoding=self.problemset_config.encoding, ) as f: return (ContentsStatus.OK, f.read()) except EnvironmentError: @@ -104,7 +105,7 @@ def get_contents(self, problem_id: str) -> Tuple[ContentsStatus, str]: raise ValueError(f"unknown mode: {mode}") def save_file(self, text: str, output_path: str): - with open(output_path, "w") as f: + with open(output_path, "w", encoding=self.problemset_config.encoding) as f: f.write(text) # 制約パラメータファイル (e.g. constraints.hpp) の作成 @@ -118,8 +119,9 @@ def create_params_file(self, problem_id: str) -> None: ext: str = Path(problem_config.params_path).suffix if ext in lang_to_class: params_maker = lang_to_class[ext]( - problem_config.constraints, - problem_config.params_path, + params=problem_config.constraints, + output_path=problem_config.params_path, + encoding=self.problemset_config.encoding, ) params_maker.run() else: diff --git a/statements_manager/src/execute_config.py b/statements_manager/src/execute_config.py index 327028b..9779e6e 100644 --- a/statements_manager/src/execute_config.py +++ b/statements_manager/src/execute_config.py @@ -183,14 +183,20 @@ def __init__(self, problemset_filename: pathlib.Path, config: dict) -> None: self.known_ids: set[str] = set() self.id_groups: list[list[str]] = list() self.problem_configs: dict[str, ProblemConfig] = dict() + self.encoding: str = self.optional( + problemset_filename, config, "encoding", "utf-8" + ) + logger.info(f"encoding = {self.encoding}") dirname = problemset_filename.parent.resolve() self.output_path = dirname / "problemset" self.template_html: str = read_text_file( - to_path(self.template.template_path), default_template_html + to_path(self.template.template_path), default_template_html, self.encoding ) self.sample_template_html: str = read_text_file( - to_path(self.template.sample_template_path), default_sample_template_html + to_path(self.template.sample_template_path), + default_sample_template_html, + self.encoding, ) def get_problem(self, id: str) -> ProblemConfig: diff --git a/statements_manager/src/params_maker/languages/cplusplus.py b/statements_manager/src/params_maker/languages/cplusplus.py index 5f03a74..eb54ec2 100644 --- a/statements_manager/src/params_maker/languages/cplusplus.py +++ b/statements_manager/src/params_maker/languages/cplusplus.py @@ -6,8 +6,8 @@ class CppParamsMaker(ParamsMaker): - def __init__(self, params: dict[str, Any], output_path: str) -> None: - super().__init__(params, output_path) + def __init__(self, params: dict[str, Any], output_path: str, encoding: str) -> None: + super().__init__(params, output_path, encoding) def header(self) -> str: return """// DO NOT EDIT THIS FILE MANUALLY diff --git a/statements_manager/src/params_maker/params_maker.py b/statements_manager/src/params_maker/params_maker.py index ab7338f..2cbb98c 100644 --- a/statements_manager/src/params_maker/params_maker.py +++ b/statements_manager/src/params_maker/params_maker.py @@ -9,9 +9,10 @@ class ParamsMaker: - def __init__(self, params: dict[str, Any], output_path: str) -> None: + def __init__(self, params: dict[str, Any], output_path: str, encoding: str) -> None: self.params = params self.output_path = output_path + self.encoding = encoding def run(self) -> None: params_lines: list[str] = [] @@ -35,7 +36,7 @@ def run(self) -> None: # if params file is the same as the existing one, do nothing params_text = "\n".join(params_lines) if pathlib.Path(self.output_path).exists(): - with open(self.output_path, "r") as f: + with open(self.output_path, "r", encoding=self.encoding) as f: reference = f.read() if params_text == reference: logger.warning( @@ -43,7 +44,7 @@ def run(self) -> None: ) return - with open(self.output_path, "w") as f: + with open(self.output_path, "w", encoding=self.encoding) as f: f.write(params_text) @abstractmethod diff --git a/statements_manager/src/render_result_cache.py b/statements_manager/src/render_result_cache.py index 6e15f52..ddadc2e 100644 --- a/statements_manager/src/render_result_cache.py +++ b/statements_manager/src/render_result_cache.py @@ -37,7 +37,7 @@ def __init__( def _load_cache(self) -> dict: if not self.cache_path.exists(): return {} - with open(self.cache_path, "r") as f: + with open(self.cache_path, "r", encoding="ascii") as f: return self._cleanup(json.load(f)) def _setup_cache(self, cache: dict) -> dict: @@ -101,7 +101,7 @@ def need_to_save(self, force_dump: bool): def save_and_check_diff(self) -> bool: json.dump( self.cache, - open(self.cache_path, "w"), + open(self.cache_path, "w", encoding="ascii"), indent=4, sort_keys=True, ) diff --git a/statements_manager/src/renderer.py b/statements_manager/src/renderer.py index 0f802ce..eb7091d 100644 --- a/statements_manager/src/renderer.py +++ b/statements_manager/src/renderer.py @@ -59,12 +59,14 @@ def __init__( self.replace_sample_format = ReplaceSampleFormatExprExtension() def replace_vars( - self, problem_config: ProblemConfig, statement_str: str | None + self, problem_config: ProblemConfig, statement_str: str | None, encoding: str ) -> str: if statement_str is None: logger.error("statement_str is None") raise RuntimeError("statement_str is None") - vars_manager = VariablesConverter(problem_config, self.sample_template_html) + vars_manager = VariablesConverter( + problem_config, self.sample_template_html, encoding + ) env = Environment( variable_start_string="{@", variable_end_string="}", @@ -188,7 +190,9 @@ def generate_html( contents = problem_config.statement.raw_text contents = self.apply_preprocess(contents) - rendered_contents = self.replace_vars(problem_config, contents) + rendered_contents = self.replace_vars( + problem_config, contents, problemset_config.encoding + ) markdown_extensions = [ self.replace_sample_format, *problem_config.statement.markdown_extensions, @@ -246,7 +250,9 @@ def generate_markdown( problem_config = problemset_config.get_problem(problem_id) if problem_config.statement.rendered_text is None: contents = problem_config.statement.raw_text - contents = self.replace_vars(problem_config, contents) + contents = self.replace_vars( + problem_config, contents, problemset_config.encoding + ) problem_config.statement.rendered_text = contents problem_config.statement.rendered_text = self.replace_assets_path( problem_config.statement.rendered_text, problem_id, is_problemset diff --git a/statements_manager/src/utils.py b/statements_manager/src/utils.py index c766b4f..c16f5f3 100644 --- a/statements_manager/src/utils.py +++ b/statements_manager/src/utils.py @@ -30,10 +30,10 @@ def read_toml_file(path: Path | None) -> dict: return toml.load(path) -def read_text_file(path: Path | None, default: str) -> str: +def read_text_file(path: Path | None, default: str, encoding: str) -> str: if path is None or not path.exists(): return default - with path.open() as f: + with path.open("r", encoding=encoding) as f: return f.read() diff --git a/statements_manager/src/variables_converter.py b/statements_manager/src/variables_converter.py index 64917ba..a045bf2 100644 --- a/statements_manager/src/variables_converter.py +++ b/statements_manager/src/variables_converter.py @@ -40,9 +40,9 @@ def to_string(value: Any, config: StatementConfig) -> str: return str(value) -def fetch_text(path: pathlib.Path) -> str: +def fetch_text(path: pathlib.Path, encoding: str) -> str: try: - return open(path).read() + return open(path, encoding=encoding).read() except OSError: return "" @@ -63,22 +63,6 @@ def convert( logger.warning("constraints are not set") -class SampleFile: - def __init__(self, filename: pathlib.Path) -> None: - self.filename = filename - - def exists(self) -> bool: - return self.filename.exists() - - def print_raw(self) -> str: - if not self.exists(): - return "" - - with open(self.filename, "r") as f: - contents = f.read() + "\n" - return contents - - class SampleData: def __init__( self, @@ -176,7 +160,11 @@ def print_warning( logger.warning(f"{sample_name}: There is no explanation.") def convert( - self, samples: dict[str, Any], problem_config: ProblemConfig, template: str + self, + samples: dict[str, Any], + problem_config: ProblemConfig, + template: str, + encoding: str, ) -> None: """ - `sample_path` が指定されていない場合: 警告を出して抜ける @@ -227,13 +215,13 @@ def convert( ) if input_file.exists(): - sample_data.input_text = fetch_text(input_file) + sample_data.input_text = fetch_text(input_file, encoding) if output_file.exists(): - sample_data.output_text = fetch_text(output_file) + sample_data.output_text = fetch_text(output_file, encoding) if md_file.exists(): - sample_data.md_text = fetch_text(md_file) + sample_data.md_text = fetch_text(md_file, encoding) if explanation_file.exists(): - sample_data.explanation_text = fetch_text(explanation_file) + sample_data.explanation_text = fetch_text(explanation_file, encoding) sample_text = env.get_template("template").render( sample_data=sample_data.to_dict(), @@ -245,11 +233,15 @@ def convert( class VariablesConverter: - def __init__(self, problem_config: ProblemConfig, sample_template: str) -> None: + def __init__( + self, problem_config: ProblemConfig, sample_template: str, encoding: str + ) -> None: self.constraints: dict = {} self.samples: dict = {} self.constraints_converter = ConstraintsConverter() self.samples_converter = SamplesConverter() self.constraints_converter.convert(self.constraints, problem_config) - self.samples_converter.convert(self.samples, problem_config, sample_template) + self.samples_converter.convert( + self.samples, problem_config, sample_template, encoding + )