1st commit

clickhouse_task/load_table.py

@@ -20,12 +20,22 @@ def load_to_clickhouse(
         log.warning(f"{table_name}: DataFrame is empty. Skipping.")
         return
 
-    arrow_table = df.to_arrow()
+    chunk_size = 10000
 
-    client.insert_arrow(
-        table=table_name,
-        arrow_table=arrow_table,
-    )
+    for start in range(0, len(df), chunk_size):
+        end = start + chunk_size
+
+        chunk_df = df.slice(start, chunk_size)
+        arrow_table = chunk_df.to_arrow()
+
+        client.insert_arrow(
+            table=table_name,
+            arrow_table=arrow_table,
+        )
+
+        log.info(
+            f"Inserted rows {start:,} to {min(end, len(df)):,}"
+        )
 
     log.info(
         f"{table_name}: inserted {len(df):,} rows into ClickHouse"