|
|
@@ -36,26 +36,51 @@ class Mysql(object):
|
|
|
def create_session(self):
|
|
|
"""创建返回一个新的数据库session"""
|
|
|
return self._DBSession()
|
|
|
-
|
|
|
+
|
|
|
+ def fetch_data_with_pagination(self, tablename, query_text, page=1, page_size=1000):
|
|
|
+ """分页查询数据"""
|
|
|
+ offset = (page - 1) * page_size # 计算偏移量
|
|
|
+ query = text(f"select {query_text} from {tablename} LIMIT :limit OFFSET :offset")
|
|
|
+ with self.create_session() as session:
|
|
|
+ results = session.execute(query, {"limit": page_size, "offset": offset}).fetchall()
|
|
|
+ df = pd.DataFrame(results)
|
|
|
+ return df
|
|
|
+
|
|
|
+ def load_data(self, tablename, query_text, page=1, page_size=1000):
|
|
|
+ # 创建一个空的DataFrame用于存储所有数据
|
|
|
+ total_df = pd.DataFrame()
|
|
|
+
|
|
|
+ try:
|
|
|
+ while True:
|
|
|
+ df = self.fetch_data_with_pagination(tablename, query_text, page, page_size)
|
|
|
+ if df.empty:
|
|
|
+ break
|
|
|
+
|
|
|
+ total_df = pd.concat([total_df, df], ignore_index=True)
|
|
|
+ print(f"Page {page}: Retrieved {len(df)} rows, Total rows so far: {len(total_df)}")
|
|
|
+ page += 1 # 继续下一页
|
|
|
+
|
|
|
+ except Exception as e:
|
|
|
+ print(f"Error: {e}")
|
|
|
+ return None
|
|
|
+
|
|
|
+ finally:
|
|
|
+ self.closed()
|
|
|
+ return total_df
|
|
|
+
|
|
|
def closed(self):
|
|
|
"""关闭连接，回收资源"""
|
|
|
self.engine.dispose()
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
+
|
|
|
client = Mysql()
|
|
|
- # 创建会话
|
|
|
- session = client.create_session()
|
|
|
+ tablename = "mock_order"
|
|
|
|
|
|
- # 使用 session 执行查询等操作
|
|
|
- try:
|
|
|
- results = session.execute(
|
|
|
- text("select * from tads_brandcul_cust_order")
|
|
|
- ).all()
|
|
|
- # 将结果转换为DataFrame
|
|
|
- df = pd.DataFrame(results).drop(columns=['stat_month']) # 提取列名
|
|
|
-
|
|
|
- # 获取c
|
|
|
- print(len(df.columns))
|
|
|
- finally:
|
|
|
- session.close()
|
|
|
+ # 设置分页参数
|
|
|
+ page = 1
|
|
|
+ page_size = 1000
|
|
|
+
|
|
|
+ query_text = '*'
|
|
|
+ client.load_data("mock_order", query_text, page, page_size)
|
Sherlock1011