联合搜索查询增加切片分批查询，避免超出数据库读取的最大限制

api_test_result.json

@@ -0,0 +1,663 @@
+{
+  "code": 200,
+  "msg": "success",
+  "data": {
+    "recommendationInfo": [
+      {
+        "id": 1,
+        "cust_code": "445381115586",
+        "recommend_score": 95.6711229475779
+      },
+      {
+        "id": 2,
+        "cust_code": "445300108811",
+        "recommend_score": 91.38608449645017
+      },
+      {
+        "id": 3,
+        "cust_code": "445323105985",
+        "recommend_score": 89.35282829559448
+      },
+      {
+        "id": 4,
+        "cust_code": "445300107560",
+        "recommend_score": 86.14051451610123
+      },
+      {
+        "id": 5,
+        "cust_code": "445381113283",
+        "recommend_score": 86.14051451610123
+      },
+      {
+        "id": 6,
+        "cust_code": "445381115818",
+        "recommend_score": 86.14051451610123
+      },
+      {
+        "id": 7,
+        "cust_code": "445381102191",
+        "recommend_score": 81.85290126617139
+      },
+      {
+        "id": 8,
+        "cust_code": "445321107670",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 9,
+        "cust_code": "445321109263",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 10,
+        "cust_code": "445323105082",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 11,
+        "cust_code": "445323105761",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 12,
+        "cust_code": "445381114446",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 13,
+        "cust_code": "445381114622",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 14,
+        "cust_code": "445381115169",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 15,
+        "cust_code": "445381115695",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 16,
+        "cust_code": "445381115892",
+        "recommend_score": 80.35615523462216
+      },
+      {
+        "id": 17,
+        "cust_code": "445300108832",
+        "recommend_score": 78.60385250602698
+      },
+      {
+        "id": 18,
+        "cust_code": "445323105901",
+        "recommend_score": 78.60385250602698
+      },
+      {
+        "id": 19,
+        "cust_code": "445321101168",
+        "recommend_score": 75.33637784008023
+      },
+      {
+        "id": 20,
+        "cust_code": "445300108768",
+        "recommend_score": 74.02591718265808
+      },
+      {
+        "id": 21,
+        "cust_code": "445323105299",
+        "recommend_score": 70.96363019242187
+      },
+      {
+        "id": 22,
+        "cust_code": "445381109407",
+        "recommend_score": 70.96363019242187
+      },
+      {
+        "id": 23,
+        "cust_code": "445300107450",
+        "recommend_score": 69.16081461517541
+      },
+      {
+        "id": 24,
+        "cust_code": "445381110702",
+        "recommend_score": 69.16081461517541
+      },
+      {
+        "id": 25,
+        "cust_code": "445300107173",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 26,
+        "cust_code": "445300108759",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 27,
+        "cust_code": "445300109062",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 28,
+        "cust_code": "445321107692",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 29,
+        "cust_code": "445321107874",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 30,
+        "cust_code": "445321108112",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 31,
+        "cust_code": "445321109012",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 32,
+        "cust_code": "445321109324",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 33,
+        "cust_code": "445321109742",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 34,
+        "cust_code": "445322106565",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 35,
+        "cust_code": "445322107763",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 36,
+        "cust_code": "445322108175",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 37,
+        "cust_code": "445323105664",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 38,
+        "cust_code": "445323105950",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 39,
+        "cust_code": "445381108007",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 40,
+        "cust_code": "445381109257",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 41,
+        "cust_code": "445381111447",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 42,
+        "cust_code": "445381111687",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 43,
+        "cust_code": "445381112908",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 44,
+        "cust_code": "445381113112",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 45,
+        "cust_code": "445381113150",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 46,
+        "cust_code": "445381113694",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 47,
+        "cust_code": "445381114073",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 48,
+        "cust_code": "445381114405",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 49,
+        "cust_code": "445381114867",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 50,
+        "cust_code": "445381115134",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 51,
+        "cust_code": "445381115168",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 52,
+        "cust_code": "445381115689",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 53,
+        "cust_code": "445381194323",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 54,
+        "cust_code": "445381194570",
+        "recommend_score": 67.13271245177921
+      },
+      {
+        "id": 55,
+        "cust_code": "445321102157",
+        "recommend_score": 59.19802703017345
+      },
+      {
+        "id": 56,
+        "cust_code": "445321105886",
+        "recommend_score": 59.19802703017345
+      },
+      {
+        "id": 57,
+        "cust_code": "445321106990",
+        "recommend_score": 55.69365037508166
+      },
+      {
+        "id": 58,
+        "cust_code": "445322107465",
+        "recommend_score": 55.69365037508166
+      },
+      {
+        "id": 59,
+        "cust_code": "445300105618",
+        "recommend_score": 51.57878322962218
+      },
+      {
+        "id": 60,
+        "cust_code": "445321106735",
+        "recommend_score": 51.57878322962218
+      },
+      {
+        "id": 61,
+        "cust_code": "445381114563",
+        "recommend_score": 51.57878322962218
+      },
+      {
+        "id": 62,
+        "cust_code": "445300107049",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 63,
+        "cust_code": "445300107263",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 64,
+        "cust_code": "445321108624",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 65,
+        "cust_code": "445322107927",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 66,
+        "cust_code": "445381112671",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 67,
+        "cust_code": "445381112722",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 68,
+        "cust_code": "445381113977",
+        "recommend_score": 46.6885925829371
+      },
+      {
+        "id": 69,
+        "cust_code": "445300106514",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 70,
+        "cust_code": "445302191673",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 71,
+        "cust_code": "445321107630",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 72,
+        "cust_code": "445321109822",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 73,
+        "cust_code": "445322107201",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 74,
+        "cust_code": "445323105827",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 75,
+        "cust_code": "445381112500",
+        "recommend_score": 40.79809819431797
+      },
+      {
+        "id": 76,
+        "cust_code": "445300105613",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 77,
+        "cust_code": "445302191818",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 78,
+        "cust_code": "445321106902",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 79,
+        "cust_code": "445321109619",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 80,
+        "cust_code": "445321109718",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 81,
+        "cust_code": "445322108189",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 82,
+        "cust_code": "445322119187",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 83,
+        "cust_code": "445322119332",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 84,
+        "cust_code": "445323104348",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 85,
+        "cust_code": "445323104448",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 86,
+        "cust_code": "445323104958",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 87,
+        "cust_code": "445323105872",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 88,
+        "cust_code": "445323105934",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 89,
+        "cust_code": "445381110788",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 90,
+        "cust_code": "445381112673",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 91,
+        "cust_code": "445381113229",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 92,
+        "cust_code": "445381113589",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 93,
+        "cust_code": "445381194893",
+        "recommend_score": 33.59704043816441
+      },
+      {
+        "id": 94,
+        "cust_code": "445300106077",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 95,
+        "cust_code": "445300106407",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 96,
+        "cust_code": "445300107612",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 97,
+        "cust_code": "445300108141",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 98,
+        "cust_code": "445300108673",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 99,
+        "cust_code": "445300109018",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 100,
+        "cust_code": "445300109033",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 101,
+        "cust_code": "445300109126",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 102,
+        "cust_code": "445321105662",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 103,
+        "cust_code": "445321106989",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 104,
+        "cust_code": "445321107457",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 105,
+        "cust_code": "445321107720",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 106,
+        "cust_code": "445321107859",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 107,
+        "cust_code": "445321107985",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 108,
+        "cust_code": "445321108505",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 109,
+        "cust_code": "445321109240",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 110,
+        "cust_code": "445321109677",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 111,
+        "cust_code": "445321109734",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 112,
+        "cust_code": "445322105327",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 113,
+        "cust_code": "445322106076",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 114,
+        "cust_code": "445322106510",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 115,
+        "cust_code": "445322106831",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 116,
+        "cust_code": "445322107613",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 117,
+        "cust_code": "445322107848",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 118,
+        "cust_code": "445322107905",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 119,
+        "cust_code": "445322107906",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 120,
+        "cust_code": "445322108178",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 121,
+        "cust_code": "445322108221",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 122,
+        "cust_code": "445322119330",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 123,
+        "cust_code": "445323104785",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 124,
+        "cust_code": "445323105741",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 125,
+        "cust_code": "445381112883",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 126,
+        "cust_code": "445381113718",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 127,
+        "cust_code": "445381113912",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 128,
+        "cust_code": "445381114616",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 129,
+        "cust_code": "445381115176",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 130,
+        "cust_code": "445381115522",
+        "recommend_score": 24.660641526955303
+      },
+      {
+        "id": 131,
+        "cust_code": "445381194454",
+        "recommend_score": 24.660641526955303
+      }
+    ]
+  }
+}

database/dao/mysql_dao.py

@@ -7,6 +7,7 @@ logger = get_logger("database.dao")
 
 class MySqlDao:
     _instance = None
+    _IN_CLAUSE_BATCH_SIZE = 1000
     
     def __new__(cls):
         if not cls._instance:
@@ -29,9 +30,15 @@ class MySqlDao:
         self._shopping_tablename = "tads_brandcul_cust_info_lbs_f"
         # self._shopping_tablename = "yunfu_shopping_mock_data"
         self._report_tablename = "tads_brandcul_report"
-        
+
         self._initialized = True
-        
+
+    def _iter_batches(self, values, batch_size=None):
+        values = [] if values is None else list(values)
+        batch_size = batch_size or self._IN_CLAUSE_BATCH_SIZE
+        for i in range(0, len(values), batch_size):
+            yield values[i:i + batch_size]
+
     def load_product_data(self, city_uuid):
         """从数据库中读取商品信息"""
         logger.info(f"Loading product data for city_uuid={city_uuid}")
@@ -127,16 +134,18 @@ class MySqlDao:
         if not cust_id_list:
             return pd.DataFrame()
 
-        query = text(f"""
-        SELECT *
-        FROM {self._cust_tablename}
-        WHERE corp_uuid = :city_uuid
-        AND cust_code IN :ids
-    """).bindparams(bindparam("ids", expanding=True))
-        params = {"city_uuid": city_uuid, "ids": list(cust_id_list)}
-        data = pd.DataFrame(self.db_helper.fetch_all(query, params))
+        results = []
+        for batch in self._iter_batches(cust_id_list):
+            query = text(f"""
+            SELECT *
+            FROM {self._cust_tablename}
+            WHERE corp_uuid = :city_uuid
+            AND cust_code IN :ids
+        """).bindparams(bindparam("ids", expanding=True))
+            params = {"city_uuid": city_uuid, "ids": batch}
+            results.append(pd.DataFrame(self.db_helper.fetch_all(query, params)))
 
-        return data
+        return pd.concat(results, ignore_index=True) if results else pd.DataFrame()
     
     def get_shop_by_ids(self, city_uuid, cust_id_list):
         """根据零售户列表查询其信息"""
@@ -144,16 +153,18 @@ class MySqlDao:
         if not cust_id_list:
             return pd.DataFrame()
 
-        query = text(f"""
-        SELECT *
-        FROM {self._shopping_tablename}
-        WHERE city_uuid = :city_uuid
-        AND cust_code IN :ids
-    """).bindparams(bindparam("ids", expanding=True))
-        params = {"city_uuid": city_uuid, "ids": list(cust_id_list)}
-        data = pd.DataFrame(self.db_helper.fetch_all(query, params))
+        results = []
+        for batch in self._iter_batches(cust_id_list):
+            query = text(f"""
+            SELECT *
+            FROM {self._shopping_tablename}
+            WHERE city_uuid = :city_uuid
+            AND cust_code IN :ids
+        """).bindparams(bindparam("ids", expanding=True))
+            params = {"city_uuid": city_uuid, "ids": batch}
+            results.append(pd.DataFrame(self.db_helper.fetch_all(query, params)))
 
-        return data
+        return pd.concat(results, ignore_index=True) if results else pd.DataFrame()
     
     def get_product_by_ids(self, city_uuid, product_id_list):
         """根据product_code列表查询其信息"""
@@ -161,11 +172,8 @@ class MySqlDao:
         if not product_id_list:
             return pd.DataFrame()
 
-        product_id_list = list(product_id_list)
-        batch_size = 2000
-        batches = [product_id_list[i:i + batch_size] for i in range(0, len(product_id_list), batch_size)]
         results = []
-        for batch in batches:
+        for batch in self._iter_batches(product_id_list):
             query = text(f"""
             SELECT *
             FROM {self._product_tablename}
@@ -184,11 +192,8 @@ class MySqlDao:
         if not product_ids:
             return pd.DataFrame()
 
-        product_ids = list(product_ids)
-        batch_size = 2000
-        batches = [product_ids[i:i + batch_size] for i in range(0, len(product_ids), batch_size)]
         results = []
-        for batch in batches:
+        for batch in self._iter_batches(product_ids):
             query = text(f"""
             SELECT *
             FROM {self._order_tablename}
@@ -213,17 +218,24 @@ class MySqlDao:
         if not cust_id_list or not product_ids:
             return pd.DataFrame()
 
-        query = text(f"""
-            SELECT cust_code, product_code, sale_qty
-            FROM {self._order_tablename}
-            WHERE city_uuid = :city_uuid
-            AND cust_code IN :cust_ids
-            AND product_code IN :product_ids
-        """).bindparams(bindparam("cust_ids", expanding=True), bindparam("product_ids", expanding=True))
-        params = {"city_uuid": city_uuid, "cust_ids": list(cust_id_list), "product_ids": list(product_ids)}
-        data = pd.DataFrame(self.db_helper.fetch_all(query, params))
+        results = []
+        for cust_batch in self._iter_batches(cust_id_list):
+            for product_batch in self._iter_batches(product_ids):
+                query = text(f"""
+                    SELECT cust_code, product_code, sale_qty
+                    FROM {self._order_tablename}
+                    WHERE city_uuid = :city_uuid
+                    AND cust_code IN :cust_ids
+                    AND product_code IN :product_ids
+                """).bindparams(bindparam("cust_ids", expanding=True), bindparam("product_ids", expanding=True))
+                params = {
+                    "city_uuid": city_uuid,
+                    "cust_ids": cust_batch,
+                    "product_ids": product_batch,
+                }
+                results.append(pd.DataFrame(self.db_helper.fetch_all(query, params)))
 
-        return data
+        return pd.concat(results, ignore_index=True) if results else pd.DataFrame()
 
     def get_order_by_product(self, city_uuid, product_id):
         logger.info(f"Getting orders by product for city_uuid={city_uuid}, product_id={product_id}")
@@ -356,4 +368,3 @@ if __name__ == "__main__":
     
     data = dao.load_order_data(city_uuid)
     print(data)
-