分页读取数据库数据

dao/mysql_client.py

@@ -36,26 +36,51 @@ class Mysql(object):
     def create_session(self):
         """创建返回一个新的数据库session"""
         return self._DBSession()
-
+    
+    def fetch_data_with_pagination(self, tablename, query_text, page=1, page_size=1000):
+        """分页查询数据"""
+        offset = (page - 1) * page_size # 计算偏移量
+        query = text(f"select {query_text} from {tablename} LIMIT :limit OFFSET :offset")
+        with self.create_session() as session:
+            results = session.execute(query, {"limit": page_size, "offset": offset}).fetchall()
+            df = pd.DataFrame(results)
+        return df
+    
+    def load_data(self, tablename, query_text, page=1, page_size=1000):
+        # 创建一个空的DataFrame用于存储所有数据
+        total_df = pd.DataFrame()
+    
+        try:
+            while True:
+                df = self.fetch_data_with_pagination(tablename, query_text, page, page_size)
+                if df.empty:
+                    break
+            
+                total_df = pd.concat([total_df, df], ignore_index=True)
+                print(f"Page {page}: Retrieved {len(df)} rows, Total rows so far: {len(total_df)}")
+                page += 1  # 继续下一页
+                
+        except Exception as e:
+            print(f"Error: {e}")
+            return None
+        
+        finally:
+            self.closed()
+            return total_df
+    
     def closed(self):
         """关闭连接，回收资源"""
         self.engine.dispose()
 
 
 if __name__ == '__main__':
+    
     client = Mysql()
-    # 创建会话
-    session = client.create_session()
+    tablename = "mock_order"
     
-    # 使用 session 执行查询等操作
-    try:
-        results = session.execute(
-            text("select * from tads_brandcul_cust_order")
-            ).all()
-        # 将结果转换为DataFrame
-        df = pd.DataFrame(results).drop(columns=['stat_month']) # 提取列名
-        
-        # 获取c
-        print(len(df.columns))
-    finally:
-        session.close()
+    # 设置分页参数
+    page = 1
+    page_size = 1000
+    
+    query_text = '*'
+    client.load_data("mock_order", query_text, page, page_size)