V0.2.2 (#25)

* Don't require PyMySQL, as it's an optional runtime dependency now

* Make logging more specific if we turn down the level from INFO to WARNING

* Don't use --noop if read-only, let's explicitly just do statistics

* Make pyyaml an optional dependency for CircleCI / tests

* Update README to note the extra dependency

* Permit Stats to work even if there are misbehaving tables

Author: jcjones

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     executor: python/default
     steps:
       - checkout
-      - run: pip install --editable .
+      - run: pip install --editable ".[pymysql]"
       - run:
           name: make test-reesults dir
           command: |

README.md

@@ -20,7 +20,7 @@ Similar tools:
  → cd mariadb-sequential-partition-manager-py
  → python3 -m venv .venv
  → . .venv/bin/activate
- → python3 -m pip install .
+ → python3 -m pip install ".[pymysql]"
  → tee /tmp/partman.conf.yml <<EOF
  partitionmanager:
   num_empty: 2
@@ -51,7 +51,7 @@ dogs:
  → cd mariadb-sequential-partition-manager-py
  → python3 -m venv .venv
  → . .venv/bin/activate
- → python3 -m pip install --editable .
+ → python3 -m pip install --editable ".[pymysql]"
  → partition-manager --log-level=debug  \
     --mariadb test_tools/fake_mariadb.sh \
     maintain --noop --table tablename

partitionmanager/cli.py

@@ -156,12 +156,14 @@ def all_configured_tables_are_compatible(conf):
     Returns True only if all are compatible, otherwise logs errors and returns
     False.
     """
+    log = logging.getLogger("all_configured_tables_are_compatible")
+
     problems = dict()
     for table in conf.tables:
         table_problems = pm_tap.get_table_compatibility_problems(conf.dbcmd, table)
         if table_problems:
             problems[table.name] = table_problems
-            logging.error(f"Cannot proceed: {table} {table_problems}")
+            log.error(f"Cannot proceed: {table} {table_problems}")
     return len(problems) == 0
 
 
@@ -282,8 +284,10 @@ def do_partition(conf):
 
     # Preflight
     if is_read_only(conf):
-        log.info("Database is read-only, forcing noop mode")
-        conf.noop = True
+        log.info("Database is read-only, only emitting statistics")
+        if conf.prometheus_stats_path:
+            do_stats(conf)
+        return dict()
 
     if not all_configured_tables_are_compatible(conf):
         return dict()
@@ -355,17 +359,22 @@ def do_partition(conf):
             )
 
     if conf.prometheus_stats_path:
-        do_stats(conf, metrics)
+        do_stats(conf, metrics=metrics)
     return all_results
 
 
 def do_stats(conf, metrics=partitionmanager.stats.PrometheusMetrics()):
     """Populates a metrics object from the tables in the configuration."""
-    if not all_configured_tables_are_compatible(conf):
-        return dict()
+
+    log = logging.getLogger("do_stats")
 
     all_results = dict()
     for table in conf.tables:
+        table_problems = pm_tap.get_table_compatibility_problems(conf.dbcmd, table)
+        if table_problems:
+            log.error(f"Cannot proceed: {table} {table_problems}")
+            continue
+
         map_data = pm_tap.get_partition_map(conf.dbcmd, table)
         statistics = partitionmanager.stats.get_statistics(
             map_data["partitions"], conf.curtime, table
@@ -433,7 +442,8 @@ def do_stats(conf, metrics=partitionmanager.stats.PrometheusMetrics()):
 def main():
     """Start here."""
     args = PARSER.parse_args()
-    logging.basicConfig(level=args.log_level)
+    log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    logging.basicConfig(level=args.log_level, format=log_format)
     if "func" not in args:
         PARSER.print_help()
         return

partitionmanager/table_append_partition.py

@@ -361,6 +361,7 @@ def _calculate_start_time(last_changed_time, evaluation_time, allowed_lifespan):
 
 
 def _plan_partition_changes(
+    table,
     partition_list,
     current_position,
     evaluation_time,
@@ -373,7 +374,7 @@ def _plan_partition_changes(
     requirements, using an estimate as to the rate of fill from the supplied
     partition_list, current_position, and evaluation_time.
     """
-    log = logging.getLogger("plan_partition_changes")
+    log = logging.getLogger(f"plan_partition_changes:{table.name}")
 
     filled_partitions, active_partition, empty_partitions = _split_partitions_around_position(
         partition_list, current_position
@@ -526,14 +527,14 @@ def _plan_partition_changes(
     return results
 
 
-def _should_run_changes(altered_partitions):
+def _should_run_changes(table, altered_partitions):
     """Returns True if the changeset should run, otherwise returns False.
 
     Evaluate the list from plan_partition_changes and determine if the set of
     changes should be performed - if all the changes are minor, they shouldn't
     be run.
     """
-    log = logging.getLogger("should_run_changes")
+    log = logging.getLogger(f"should_run_changes:{table.name}")
 
     for p in altered_partitions:
         if isinstance(p, partitionmanager.types.NewPlannedPartition):
@@ -650,17 +651,20 @@ def get_pending_sql_reorganize_partition_commands(
         algorithm is running.
     """
 
-    log = logging.getLogger("get_pending_sql_reorganize_partition_commands")
+    log = logging.getLogger(
+        f"get_pending_sql_reorganize_partition_commands:{table.name}"
+    )
 
     partition_changes = _plan_partition_changes(
+        table,
         partition_list,
         current_position,
         evaluation_time,
         allowed_lifespan,
         num_empty_partitions,
     )
 
-    if not _should_run_changes(partition_changes):
+    if not _should_run_changes(table, partition_changes):
         log.info(f"{table} does not need to be modified currently.")
         return list()
 

partitionmanager/table_append_partition_test.py

@@ -497,6 +497,7 @@ def test_predict_forward_time(self):
     def test_plan_partition_changes_no_empty_partitions(self):
         with self.assertRaises(NoEmptyPartitionsAvailableException):
             _plan_partition_changes(
+                Table("table"),
                 [mkPPart("p_20201231", 0), mkPPart("p_20210102", 200)],
                 mkPos(50),
                 datetime(2021, 1, 1, tzinfo=timezone.utc),
@@ -505,8 +506,9 @@ def test_plan_partition_changes_no_empty_partitions(self):
             )
 
     def test_plan_partition_changes_imminent(self):
-        with self.assertLogs("plan_partition_changes", level="INFO") as logctx:
+        with self.assertLogs("plan_partition_changes:table", level="INFO") as logctx:
             planned = _plan_partition_changes(
+                Table("table"),
                 [
                     mkPPart("p_20201231", 100),
                     mkPPart("p_20210102", 200),
@@ -521,7 +523,7 @@ def test_plan_partition_changes_imminent(self):
         self.assertEqual(
             logctx.output,
             [
-                "INFO:plan_partition_changes:Start-of-fill predicted at "
+                "INFO:plan_partition_changes:table:Start-of-fill predicted at "
                 "2021-01-03 which is not 2021-01-02. This change will be marked "
                 "as important to ensure that p_20210102: (200) is moved to "
                 "2021-01-03"
@@ -545,8 +547,9 @@ def test_plan_partition_changes_imminent(self):
         )
 
     def test_plan_partition_changes_wildly_off_dates(self):
-        with self.assertLogs("plan_partition_changes", level="INFO") as logctx:
+        with self.assertLogs("plan_partition_changes:table", level="INFO") as logctx:
             planned = _plan_partition_changes(
+                Table("table"),
                 [
                     mkPPart("p_20201231", 100),
                     mkPPart("p_20210104", 200),
@@ -561,7 +564,7 @@ def test_plan_partition_changes_wildly_off_dates(self):
         self.assertEqual(
             logctx.output,
             [
-                "INFO:plan_partition_changes:Start-of-fill predicted at "
+                "INFO:plan_partition_changes:table:Start-of-fill predicted at "
                 "2021-01-02 which is not 2021-01-04. This change will be marked "
                 "as important to ensure that p_20210104: (200) is moved to "
                 "2021-01-02"
@@ -583,6 +586,7 @@ def test_plan_partition_changes_wildly_off_dates(self):
 
     def test_plan_partition_changes_long_delay(self):
         planned = _plan_partition_changes(
+            Table("table"),
             [
                 mkPPart("p_20210101", 100),
                 mkPPart("p_20210415", 200),
@@ -610,6 +614,7 @@ def test_plan_partition_changes_long_delay(self):
     def test_plan_partition_changes_short_names(self):
         self.maxDiff = None
         planned = _plan_partition_changes(
+            Table("table"),
             [
                 mkPPart("p_2019", 1912499867),
                 mkPPart("p_2020", 8890030931),
@@ -651,6 +656,7 @@ def test_plan_partition_changes_short_names(self):
 
     def test_plan_partition_changes_bespoke_names(self):
         planned = _plan_partition_changes(
+            Table("table"),
             [mkPPart("p_start", 100), mkTailPart("p_future")],
             mkPos(50),
             datetime(2021, 1, 6, tzinfo=timezone.utc),
@@ -686,6 +692,7 @@ def test_plan_partition_changes_bespoke_names(self):
     def test_plan_partition_changes(self):
         self.maxDiff = None
         planned = _plan_partition_changes(
+            Table("table"),
             [
                 mkPPart("p_20201231", 100),
                 mkPPart("p_20210102", 200),
@@ -710,6 +717,7 @@ def test_plan_partition_changes(self):
 
         self.assertEqual(
             _plan_partition_changes(
+                Table("table"),
                 [
                     mkPPart("p_20201231", 100),
                     mkPPart("p_20210102", 200),
@@ -739,6 +747,7 @@ def test_plan_partition_changes_misprediction(self):
         match reality. """
         self.maxDiff = None
         planned = _plan_partition_changes(
+            Table("table"),
             [
                 mkPPart("p_20210505", 9505010028),
                 mkPPart("p_20210604", 10152257517),
@@ -770,25 +779,32 @@ def test_plan_partition_changes_misprediction(self):
     def test_should_run_changes(self):
         self.assertFalse(
             _should_run_changes(
-                [ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position([300])]
+                Table("table"),
+                [
+                    ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position(
+                        [300]
+                    )
+                ],
             )
         )
 
         self.assertFalse(
             _should_run_changes(
+                Table("table"),
                 [
                     ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position(
                         [300]
                     ),
                     ChangePlannedPartition(mkPPart("p_20210109", 1000)).set_position(
                         [1300]
                     ),
-                ]
+                ],
             )
         )
-        with self.assertLogs("should_run_changes", level="DEBUG") as logctx:
+        with self.assertLogs("should_run_changes:table", level="DEBUG") as logctx:
             self.assertTrue(
                 _should_run_changes(
+                    Table("table"),
                     [
                         ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position(
                             [302]
@@ -802,17 +818,21 @@ def test_should_run_changes(self):
                         NewPlannedPartition()
                         .set_position([662])
                         .set_timestamp(datetime(2021, 1, 23, tzinfo=timezone.utc)),
-                    ]
+                    ],
                 )
             )
         self.assertEqual(
             logctx.output,
-            ["DEBUG:should_run_changes:Add: [542] 2021-01-16 " "00:00:00+00:00 is new"],
+            [
+                "DEBUG:should_run_changes:table:Add: [542] 2021-01-16 "
+                "00:00:00+00:00 is new"
+            ],
         )
 
-        with self.assertLogs("should_run_changes", level="DEBUG") as logctx:
+        with self.assertLogs("should_run_changes:table", level="DEBUG") as logctx:
             self.assertTrue(
                 _should_run_changes(
+                    Table("table"),
                     [
                         ChangePlannedPartition(mkPPart("p_20210102", 200)),
                         NewPlannedPartition()
@@ -821,12 +841,15 @@ def test_should_run_changes(self):
                         NewPlannedPartition()
                         .set_position([662])
                         .set_timestamp(datetime(2021, 1, 23, tzinfo=timezone.utc)),
-                    ]
+                    ],
                 )
             )
         self.assertEqual(
             logctx.output,
-            ["DEBUG:should_run_changes:Add: [542] 2021-01-16 " "00:00:00+00:00 is new"],
+            [
+                "DEBUG:should_run_changes:table:Add: [542] 2021-01-16 "
+                "00:00:00+00:00 is new"
+            ],
         )
 
     def testgenerate_sql_reorganize_partition_commands_no_change(self):
@@ -976,6 +999,7 @@ def test_plan_andgenerate_sql_reorganize_partition_commands_with_future_partitio
         self
     ):
         planned = _plan_partition_changes(
+            Table("table"),
             [
                 mkPPart("p_20201231", 100),
                 mkPPart("p_20210104", 200),
@@ -999,7 +1023,7 @@ def test_plan_andgenerate_sql_reorganize_partition_commands_with_future_partitio
 
     def test_get_pending_sql_reorganize_partition_commands_no_changes(self):
         with self.assertLogs(
-            "get_pending_sql_reorganize_partition_commands", level="INFO"
+            "get_pending_sql_reorganize_partition_commands:plushies", level="INFO"
         ) as logctx:
             cmds = get_pending_sql_reorganize_partition_commands(
                 table=Table("plushies"),
@@ -1017,7 +1041,7 @@ def test_get_pending_sql_reorganize_partition_commands_no_changes(self):
         self.assertEqual(
             logctx.output,
             [
-                "INFO:get_pending_sql_reorganize_partition_commands:"
+                "INFO:get_pending_sql_reorganize_partition_commands:plushies:"
                 "Table plushies does not need to be modified currently."
             ],
         )
@@ -1026,7 +1050,7 @@ def test_get_pending_sql_reorganize_partition_commands_no_changes(self):
 
     def test_get_pending_sql_reorganize_partition_commands_with_changes(self):
         with self.assertLogs(
-            "get_pending_sql_reorganize_partition_commands", level="DEBUG"
+            "get_pending_sql_reorganize_partition_commands:plushies", level="DEBUG"
         ) as logctx:
             cmds = get_pending_sql_reorganize_partition_commands(
                 table=Table("plushies"),
@@ -1044,7 +1068,7 @@ def test_get_pending_sql_reorganize_partition_commands_with_changes(self):
         self.assertEqual(
             logctx.output,
             [
-                "DEBUG:get_pending_sql_reorganize_partition_commands:"
+                "DEBUG:get_pending_sql_reorganize_partition_commands:plushies:"
                 "Table plushies has changes waiting."
             ],
         )

setup.py

@@ -19,7 +19,8 @@
     zip_safe=False,
     include_package_data=True,
     python_requires=">=3.6",
-    install_requires=["PyMySQL >= 1.0.2", "pyyaml"],
+    install_requires=["pyyaml"],
+    extras_require={"pymysql": ["PyMySQL >= 1.0.2"]},
     packages=["partitionmanager"],
     entry_points={"console_scripts": ["partition-manager=partitionmanager.cli:main"]},
 )