sivethe · Jan 30, 2025 · Jan 28, 2025 · Jan 28, 2025 · Jan 28, 2025 · Jan 28, 2025
@@ -139,4 +139,8 @@ jobs:
 
       - name: Citus Indent
         run: |
-          citus_indent --check || (echo ""; echo "citus_indent failed, refer to CONTRIBUTING.md on how to use citus_indent"; exit 1)    
+          citus_indent --check || (echo ""; echo "citus_indent failed, refer to CONTRIBUTING.md on how to use citus_indent"; exit 1)
+
+      - name: Validate sanity of files
+        run: |
+          ./scripts/validate_extension_file_state.sh
@@ -1,5 +1,5 @@
 comment = 'Multi-Node API surface for DocumentDB'
-default_version = '0.100-0'
+default_version = '0.101-0'
 module_pathname = '$libdir/pg_documentdb_distributed'
 relocatable = false
 superuser = true

@@ -437,6 +437,7 @@ NOTICE:  creating collection
 
 SELECT * FROM aggregate_cursor_first_page('newdb', '{ "aggregate": "source", "pipeline": [ {"$project" : {"_id" : 0} } ,{"$out" : "target"  } ], "cursor": { "batchSize": 1 } }', 4294967294);
 NOTICE:  creating collection
+NOTICE:  There are open held portals. Closing them
                                                             cursorpage                                                             | continuation | persistconnection | cursorid 
 ---------------------------------------------------------------------
  { "cursor" : { "id" : { "$numberLong" : "0" }, "ns" : "newdb.source", "firstBatch" : [  ] }, "ok" : { "$numberDouble" : "1.0" } } |              | f                 |        0
@@ -673,7 +674,7 @@ NOTICE:  creating collection
 
 EXPLAIN (VERBOSE ON, COSTS OFF) SELECT document FROM bson_aggregation_pipeline('db', '{ "aggregate": "explainsrc", "pipeline": [{"$out" :  "target"} ] }');
 NOTICE:  creating collection
-                                                                                                                                                                                                         QUERY PLAN                                                                                                                                                                                                         
+                                                                                                                                                                                                              QUERY PLAN                                                                                                                                                                                                              
 ---------------------------------------------------------------------
  Custom Scan (Citus MERGE INTO ...)
    MERGE INTO documents_9864 method: pull to coordinator
@@ -699,7 +700,7 @@ SELECT documentdb_api.shard_collection('newdb', 'explainsrc', '{ "a": "hashed" }
 (1 row)
 
 EXPLAIN (VERBOSE ON, COSTS OFF) SELECT document FROM bson_aggregation_pipeline('db', '{ "aggregate": "explainsrc", "pipeline": [{"$out" :  "target"} ] }');
-                                                                                                                                                                                                         QUERY PLAN                                                                                                                                                                                                         
+                                                                                                                                                                                                              QUERY PLAN                                                                                                                                                                                                              
 ---------------------------------------------------------------------
  Custom Scan (Citus MERGE INTO ...)
    MERGE INTO documents_9864 method: pull to coordinator

@@ -309,14 +309,16 @@ EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bs
 (3 rows)
 
 -- shard the collection on a, should sort on object_id only when there is a shard filter.
-SELECT documentdb_api.shard_collection('sort_pushdown', 'coll', '{"a": "hashed"}', false);
+SELECT documentdb_api.shard_collection('{ "shardCollection": "sort_pushdown.coll", "key": { "a": "hashed" }, "numInitialChunks": 2 }');
  shard_collection 
 ---------------------------------------------------------------------
 
 (1 row)
 
+SET citus.explain_all_tasks to on;
+SET citus.max_adaptive_executor_pool_size to 1;
 EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bson_aggregation_find('sort_pushdown', '{ "find": "coll", "filter": {"a": {"$eq": 14}}, "sort": {"_id": 1}, "limit": 20 }');
-                                                                                           QUERY PLAN                                                                                           
+                                                                                                QUERY PLAN                                                                                                
 ---------------------------------------------------------------------
  Custom Scan (Citus Adaptive) (actual rows=20 loops=1)
    Task Count: 1
@@ -329,9 +331,9 @@ EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bs
                ->  Sort (actual rows=20 loops=1)
                      Sort Key: object_id
                      Sort Method: top-N heapsort  Memory: 26kB
-                     ->  Bitmap Heap Scan on documents_964001_9640005 collection (actual rows=3334 loops=1)
+                     ->  Bitmap Heap Scan on documents_964001_9640004 collection (actual rows=3334 loops=1)
                            Recheck Cond: ((document OPERATOR(documentdb_api_catalog.@=) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson) AND (shard_key_value = '3904677705083871083'::bigint))
-                           Heap Blocks: exact=69
+                           Heap Blocks: exact=104
                            ->  BitmapAnd (actual rows=0 loops=1)
                                  ->  Bitmap Index Scan on a_id (actual rows=3334 loops=1)
                                        Index Cond: (document OPERATOR(documentdb_api_catalog.@=) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson)
@@ -340,16 +342,16 @@ EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bs
 (19 rows)
 
 EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bson_aggregation_find('sort_pushdown', '{ "find": "coll", "filter": {"a": {"$gt": 14}}, "sort": {"_id": 1}, "limit": 20 }');
-                                                                       QUERY PLAN                                                                        
+                                                                            QUERY PLAN                                                                             
 ---------------------------------------------------------------------
  Limit (actual rows=20 loops=1)
    ->  Sort (actual rows=20 loops=1)
          Sort Key: remote_scan."?sort?" NULLS FIRST
          Sort Method: quicksort  Memory: 26kB
          ->  Custom Scan (Citus Adaptive) (actual rows=20 loops=1)
-               Task Count: 8
+               Task Count: 2
                Tuple data received from nodes: 700 bytes
-               Tasks Shown: One of 8
+               Tasks Shown: All
                ->  Task
                      Tuple data received from node: 700 bytes
                      Node: host=localhost port=58070 dbname=regression
@@ -359,32 +361,51 @@ EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bs
                                  Sort Method: top-N heapsort  Memory: 26kB
                                  ->  Bitmap Heap Scan on documents_964001_9640004 collection (actual rows=3333 loops=1)
                                        Recheck Cond: (document OPERATOR(documentdb_api_catalog.@>) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson)
-                                       Heap Blocks: exact=35
+                                       Heap Blocks: exact=104
                                        ->  Bitmap Index Scan on a_id (actual rows=3333 loops=1)
                                              Index Cond: (document OPERATOR(documentdb_api_catalog.@>) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson)
-(20 rows)
+               ->  Task
+                     Tuple data received from node: 0 bytes
+                     Node: host=localhost port=58070 dbname=regression
+                     ->  Limit (actual rows=0 loops=1)
+                           ->  Sort (actual rows=0 loops=1)
+                                 Sort Key: (documentdb_api_catalog.bson_orderby(document, '{ "_id" : { "$numberInt" : "1" } }'::documentdb_core.bson)) NULLS FIRST
+                                 Sort Method: quicksort  Memory: 25kB
+                                 ->  Bitmap Heap Scan on documents_964001_9640005 collection (actual rows=0 loops=1)
+                                       Recheck Cond: (document OPERATOR(documentdb_api_catalog.@>) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson)
+                                       ->  Bitmap Index Scan on a_id (actual rows=0 loops=1)
+                                             Index Cond: (document OPERATOR(documentdb_api_catalog.@>) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson)
+(31 rows)
 
 -- no filter on sharded collection should not sort on object_id
 EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bson_aggregation_find('sort_pushdown', '{ "find": "coll", "filter": {}, "sort": {"_id": 1}, "limit": 20 }');
-                                                                       QUERY PLAN                                                                        
+                                                                            QUERY PLAN                                                                             
 ---------------------------------------------------------------------
  Limit (actual rows=20 loops=1)
    ->  Sort (actual rows=20 loops=1)
          Sort Key: remote_scan."?sort?" NULLS FIRST
-         Sort Method: quicksort  Memory: 28kB
-         ->  Custom Scan (Citus Adaptive) (actual rows=40 loops=1)
-               Task Count: 8
-               Tuple data received from nodes: 1400 bytes
-               Tasks Shown: One of 8
+         Sort Method: quicksort  Memory: 26kB
+         ->  Custom Scan (Citus Adaptive) (actual rows=20 loops=1)
+               Task Count: 2
+               Tuple data received from nodes: 700 bytes
+               Tasks Shown: All
                ->  Task
                      Tuple data received from node: 700 bytes
                      Node: host=localhost port=58070 dbname=regression
                      ->  Limit (actual rows=20 loops=1)
                            ->  Sort (actual rows=20 loops=1)
                                  Sort Key: (documentdb_api_catalog.bson_orderby(document, '{ "_id" : { "$numberInt" : "1" } }'::documentdb_core.bson)) NULLS FIRST
                                  Sort Method: top-N heapsort  Memory: 26kB
-                                 ->  Seq Scan on documents_964001_9640005 collection (actual rows=6667 loops=1)
-(16 rows)
+                                 ->  Seq Scan on documents_964001_9640004 collection (actual rows=10000 loops=1)
+               ->  Task
+                     Tuple data received from node: 0 bytes
+                     Node: host=localhost port=58070 dbname=regression
+                     ->  Limit (actual rows=0 loops=1)
+                           ->  Sort (actual rows=0 loops=1)
+                                 Sort Key: (documentdb_api_catalog.bson_orderby(document, '{ "_id" : { "$numberInt" : "1" } }'::documentdb_core.bson)) NULLS FIRST
+                                 Sort Method: quicksort  Memory: 25kB
+                                 ->  Seq Scan on documents_964001_9640005 collection (actual rows=0 loops=1)
+(24 rows)
 
 -- drop compound index, should use the _id index
 CALL documentdb_api.drop_indexes('sort_pushdown', '{ "dropIndexes": "coll", "index": "a_id"}');
@@ -395,7 +416,7 @@ CALL documentdb_api.drop_indexes('sort_pushdown', '{ "dropIndexes": "coll", "ind
 
 ANALYZE documentdb_data.documents_964001;
 EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bson_aggregation_find('sort_pushdown', '{ "find": "coll", "filter": {"a": {"$eq": 14}}, "sort": {"_id": 1}, "limit": 20 }');
-                                                         QUERY PLAN                                                         
+                                                              QUERY PLAN                                                              
 ---------------------------------------------------------------------
  Custom Scan (Citus Adaptive) (actual rows=20 loops=1)
    Task Count: 1
@@ -405,32 +426,30 @@ EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bs
          Tuple data received from node: 420 bytes
          Node: host=localhost port=58070 dbname=regression
          ->  Limit (actual rows=20 loops=1)
-               ->  Index Scan using _id_ on documents_964001_9640005 collection (actual rows=20 loops=1)
+               ->  Index Scan using _id_ on documents_964001_9640004 collection (actual rows=20 loops=1)
                      Index Cond: (shard_key_value = '3904677705083871083'::bigint)
                      Filter: (document OPERATOR(documentdb_api_catalog.@=) '{ "a" : { "$numberInt" : "14" } }'::documentdb_core.bson)
 (11 rows)
 
 -- or should push down to the shards and use object_id
 EXPLAIN (COSTS OFF, TIMING OFF, ANALYZE ON, SUMMARY OFF) SELECT document FROM bson_aggregation_find('sort_pushdown', '{ "find": "coll", "filter": {"$or": [{"a": {"$eq": 14}}, {"a": {"$eq": 22}}]}, "sort": {"_id": 1}, "limit": 20 }');
-                                                                                                                                         QUERY PLAN                                                                                                                                         
+                                                                                                                                        QUERY PLAN                                                                                                                                        
 ---------------------------------------------------------------------
- Limit (actual rows=20 loops=1)
-   ->  Sort (actual rows=20 loops=1)
-         Sort Key: remote_scan."?sort?"
-         Sort Method: quicksort  Memory: 28kB
-         ->  Custom Scan (Citus Adaptive) (actual rows=40 loops=1)
-               Task Count: 2
-               Tuple data received from nodes: 1280 bytes
-               Tasks Shown: One of 2
-               ->  Task
-                     Tuple data received from node: 640 bytes
-                     Node: host=localhost port=58070 dbname=regression
-                     ->  Limit (actual rows=20 loops=1)
-                           ->  Sort (actual rows=20 loops=1)
-                                 Sort Key: object_id
-                                 Sort Method: top-N heapsort  Memory: 26kB
-                                 ->  Seq Scan on documents_964001_9640005 collection (actual rows=3334 loops=1)
-                                       Filter: ((document OPERATOR(documentdb_api_catalog.@*=) '{ "a" : [ { "$numberInt" : "14" }, { "$numberInt" : "22" } ] }'::documentdb_core.bson) AND ((shard_key_value = '3904677705083871083'::bigint) OR (shard_key_value = '-8641956760761651950'::bigint)))
-                                       Rows Removed by Filter: 3333
-(18 rows)
+ Custom Scan (Citus Adaptive) (actual rows=20 loops=1)
+   Task Count: 1
+   Tuple data received from nodes: 420 bytes
+   Tasks Shown: All
+   ->  Task
+         Tuple data received from node: 420 bytes
+         Node: host=localhost port=58070 dbname=regression
+         ->  Limit (actual rows=20 loops=1)
+               ->  Sort (actual rows=20 loops=1)
+                     Sort Key: object_id
+                     Sort Method: top-N heapsort  Memory: 26kB
+                     ->  Seq Scan on documents_964001_9640004 collection (actual rows=6667 loops=1)
+                           Filter: ((document OPERATOR(documentdb_api_catalog.@*=) '{ "a" : [ { "$numberInt" : "14" }, { "$numberInt" : "22" } ] }'::documentdb_core.bson) AND ((shard_key_value = '3904677705083871083'::bigint) OR (shard_key_value = '-8641956760761651950'::bigint)))
+                           Rows Removed by Filter: 3333
+(14 rows)
 
+RESET citus.explain_all_tasks;
+RESET citus.max_adaptive_executor_pool_size;