safern · Feb 26, 2025 · Feb 19, 2025 · Feb 19, 2025 · Feb 20, 2025 · Feb 21, 2025
@@ -3,6 +3,9 @@
 * Support exact search for vector search queries *[Feature]*
 * Inline $match with let in $lookup pipelines as JOIN Filter *[Perf]*
 * Support TTL indexes *[Bugfix]* (#34)
+* Support joining between postgres and documentdb tables *[Feature]* (#61)
+* Support current_op command *[Feature]* (#59)
+* Support for list_databases command *[Feature]* (#45)
 
 ### documentdb v0.101-0 (February 12, 2025) ###
 * Push $graphlookup recursive CTE JOIN filters to index *[Perf]*

@@ -6,6 +6,10 @@ Makefile:;
 
 .PHONY: %
 
+check-no-distributed:
+	$(MAKE) -C pg_documentdb_core check
+	$(MAKE) -C pg_documentdb check
+
 .DEFAULT:
 	$(MAKE) -C pg_documentdb_core
 	$(MAKE) -C pg_documentdb

@@ -503,4 +503,5 @@ OrdinalPosition,ErrorName
 502,Location7750301
 503,Location7750302
 504,Location7750303
-505,Location51003
+505,Location51003
+506,Location7749501
@@ -500,6 +500,7 @@ ErrorMapping,ErrorName
 6586400,Location6586400
 7429703,Location7429703
 7436100,Location7436100
+7749501,Location7749501
 7750301,Location7750301
 7750302,Location7750302
 7750303,Location7750303

@@ -39,6 +39,12 @@ char *DistributedExtensionName = "documentdb_distributed";
 bool CreateDistributedFunctions = false;
 bool CreateIndexBuildQueueTable = false;
 
+typedef struct ClusterOperationVersions
+{
+	ExtensionVersion InstalledVersion;
+	ExtensionVersion LastUpgradeVersion;
+} ClusterOperationVersions;
+
 extern char * GetIndexQueueName(void);
 
 static char * GetClusterInitializedVersion(void);
@@ -65,6 +71,7 @@ static void GetInstalledVersion(ExtensionVersion *installedVersion);
 static void ParseVersionString(ExtensionVersion *extensionVersion, char *versionString);
 static bool SetupCluster(bool isInitialize);
 static void SetPermissionsForReadOnlyRole(void);
+static void CheckAndReplicateReferenceTable(const char *schema, const char *tableName);
 static ArrayType * GetCollectionIds(void);
 
 PG_FUNCTION_INFO_V1(command_initialize_cluster);
@@ -118,12 +125,21 @@ command_complete_upgrade(PG_FUNCTION_ARGS)
  * than the current installed version.
  */
 static inline bool
-ShouldRunSetupForVersion(ExtensionVersion lastUpgradeVersion,
-						 ExtensionVersion installedVersion,
+ShouldRunSetupForVersion(ClusterOperationVersions *versions,
 						 MajorVersion major, int minor, int patch)
 {
-	return !IsExtensionVersionAtleast(lastUpgradeVersion, major, minor, patch) &&
-		   IsExtensionVersionAtleast(installedVersion, major, minor, patch);
+	return !IsExtensionVersionAtleast(versions->LastUpgradeVersion, major, minor,
+									  patch) &&
+		   IsExtensionVersionAtleast(versions->InstalledVersion, major, minor, patch);
+}
+
+
+static bool
+ShouldRunSetupForVersionForHook(void *versionsVoid,
+								int major, int minor, int patch)
+{
+	return ShouldRunSetupForVersion((ClusterOperationVersions *) versionsVoid,
+									(MajorVersion) major, minor, patch);
 }
 
 
@@ -163,7 +179,16 @@ SetupCluster(bool isInitialize)
 					installedVersion.Major, installedVersion.Minor,
 					installedVersion.Patch));
 	}
-	else
+
+	ClusterOperationVersions versions =
+	{
+		.InstalledVersion = installedVersion,
+		.LastUpgradeVersion = lastUpgradeVersion
+	};
+
+	/* If the version is < 0.0-5 or if it's an initialize ensure metadata collections are created */
+	if (isInitialize ||
+		ShouldRunSetupForVersion(&versions, DocDB_V0, 0, 5))
 	{
 		/*
 		 * We should only create and modify schema objects here for versions that are no longer covered by the upgrade path.
@@ -194,19 +219,19 @@ SetupCluster(bool isInitialize)
 	/*
 	 * For initialize, lastUpgradeVersion will always be 1.4-0, so all of the below conditions will apply if the installedVersion meets the requirement.
 	 */
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 7, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 7, 0))
 	{
 		AddCollectionsTableViewDefinition();
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 7, 0) &&
-		!ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 12, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 7, 0) &&
+		!ShouldRunSetupForVersion(&versions, DocDB_V0, 12, 0))
 	{
 		/* Schedule happens again at 1.12 */
 		ScheduleIndexBuildTasks(ExtensionObjectPrefix);
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 8, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 8, 0))
 	{
 		CreateExtensionVersionsTrigger();
 
@@ -215,13 +240,13 @@ SetupCluster(bool isInitialize)
 		AddCollectionsTableValidationColumns();
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 12, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 12, 0))
 	{
 		CreateIndexBuildsTable();
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 12, 0) &&
-		!ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 15, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 12, 0) &&
+		!ShouldRunSetupForVersion(&versions, DocDB_V0, 15, 0))
 	{
 		/* Unschedule index tasks from old queue. */
 		char *oldExtensionPrefix = ExtensionObjectPrefix;
@@ -231,26 +256,26 @@ SetupCluster(bool isInitialize)
 		ScheduleIndexBuildTasks(extensionPrefix);
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 14, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 14, 0))
 	{
 		DropLegacyChangeStream();
 		AddUserColumnsToIndexQueue();
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 15, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 15, 0))
 	{
 		/* reduce the Index background cron job schedule to 2 seconds by default. */
 		char *extensionPrefix = ExtensionObjectPrefixV2;
 		UnscheduleIndexBuildTasks(extensionPrefix);
 		ScheduleIndexBuildTasks(extensionPrefix);
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 17, 1))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 17, 1))
 	{
 		SetPermissionsForReadOnlyRole();
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 21, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 21, 0))
 	{
 		if (!isInitialize && ClusterAdminRole[0] != '\0')
 		{
@@ -266,7 +291,7 @@ SetupCluster(bool isInitialize)
 		}
 	}
 
-	if (ShouldRunSetupForVersion(lastUpgradeVersion, installedVersion, DocDB_V0, 23, 0))
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 23, 0))
 	{
 		/* Re-add the primary key in the context of the cluster operations. */
 		StringInfo cmdStr = makeStringInfo();
@@ -286,8 +311,18 @@ SetupCluster(bool isInitialize)
 									&isNull);
 	}
 
+	if (ShouldRunSetupForVersion(&versions, DocDB_V0, 23, 2))
+	{
+		CheckAndReplicateReferenceTable(ApiCatalogSchemaName, "collections");
+		CheckAndReplicateReferenceTable(ApiCatalogSchemaName, "collection_indexes");
+
+		StringInfo relationName = makeStringInfo();
+		appendStringInfo(relationName, "%s_cluster_data", ExtensionObjectPrefix);
+		CheckAndReplicateReferenceTable(ApiDistributedSchemaName, relationName->data);
+	}
+
 	/* we call the post setup cluster hook to allow the extension to do any additional setup */
-	PostSetupClusterHook(isInitialize);
+	PostSetupClusterHook(isInitialize, &ShouldRunSetupForVersionForHook, &versions);
 
 	TriggerInvalidateClusterMetadata();
 	return true;
@@ -348,8 +383,9 @@ DistributeCrudFunctions()
 
 	/* Push down the delete/insert/update one function calls */
 	appendStringInfo(relationName,
-					 "%s.delete_one(bigint,bigint,bson,bson,bool,bson,text)",
-					 ApiInternalSchemaName);
+					 "%s.delete_one(bigint,bigint,%s,%s,bool,%s,text)",
+					 ApiInternalSchemaName, FullBsonTypeName, FullBsonTypeName,
+					 FullBsonTypeName);
 	CreateDistributedFunction(
 		relationName->data,
 		distributionArgName,
@@ -358,8 +394,8 @@ DistributeCrudFunctions()
 		);
 
 	resetStringInfo(relationName);
-	appendStringInfo(relationName, "%s.insert_one(bigint,bigint,bson,text)",
-					 ApiInternalSchemaName);
+	appendStringInfo(relationName, "%s.insert_one(bigint,bigint,%s,text)",
+					 ApiInternalSchemaName, FullBsonTypeName);
 	CreateDistributedFunction(
 		relationName->data,
 		distributionArgName,
@@ -369,8 +405,10 @@ DistributeCrudFunctions()
 
 	resetStringInfo(relationName);
 	appendStringInfo(relationName,
-					 "%s.update_one(bigint,bigint,bson,bson,bson,bool,bson,bool,bson,bson,text)",
-					 ApiInternalSchemaName);
+					 "%s.update_one(bigint,bigint,%s,%s,%s,bool,%s,bool,%s,%s,text)",
+					 ApiInternalSchemaName, FullBsonTypeName, FullBsonTypeName,
+					 FullBsonTypeName,
+					 FullBsonTypeName, FullBsonTypeName, FullBsonTypeName);
 	CreateDistributedFunction(
 		relationName->data,
 		distributionArgName,
@@ -966,6 +1004,39 @@ SetPermissionsForReadOnlyRole()
 }
 
 
+/*
+ * Checks if the given reference table is replicated.
+ */
+static void
+CheckAndReplicateReferenceTable(const char *schema, const char *tableName)
+{
+	StringInfo relationName = makeStringInfo();
+	appendStringInfo(relationName, "%s_%s", ExtensionObjectPrefix, "cluster_data");
+	if (!(strcmp(tableName, "collections") == 0 ||
+		  strcmp(tableName, "collection_indexes") == 0 ||
+		  strcmp(tableName, relationName->data) == 0))
+	{
+		return;
+	}
+
+	StringInfo queryStringInfo = makeStringInfo();
+	appendStringInfo(queryStringInfo,
+					 "SELECT shardid FROM pg_catalog.pg_dist_shard WHERE logicalrelid = '%s.%s'::regclass",
+					 schema, tableName);
+
+	bool isNull = false;
+	ExtensionExecuteQueryViaSPI(queryStringInfo->data, false, SPI_OK_SELECT, &isNull);
+
+	if (isNull)
+	{
+		/* Not replicated, replicate it.*/
+		resetStringInfo(relationName);
+		appendStringInfo(relationName, "%s.%s", schema, tableName);
+		CreateReferenceTable(relationName->data);
+	}
+}
+
+
 /*
  * Gets the collection Ids where view_definition is NULL
  */

@@ -11,6 +11,8 @@ export PG_CONFIG
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
+pg_major_version := $(shell $(PG_CONFIG) --version | awk -F' ' '{ print $$2 }' | awk -F'.' '{ print $$1 }')
+
 EXTENSIONLOAD :=  --load-extension=tsm_system_rows --load-extension=pg_cron --load-extension=vector --load-extension=postgis --load-extension citus  --load-extension=documentdb_core --load-extension=rum --load-extension=documentdb --load-extension=documentdb_distributed
 
 MAKEFILE_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
@@ -22,20 +24,26 @@ export PGISOLATIONTIMEOUT = 60
 .PHONY: check-basic check-dist-minimal
 
 define common_test
-	$(top_builddir)/src/test/regress/pg_regress --dlpath=$(BASEPATH) $(EXTENSIONLOAD) --temp-instance ./tmp --temp-config ./postgresql.conf --host localhost --port 58070 $(1) $(2) || (cat regression.diffs && false)
+	$(top_builddir)/src/test/regress/pg_regress --encoding=UTF8 --dlpath=$(BASEPATH) $(EXTENSIONLOAD) --temp-instance ./tmp --temp-config ./postgresql.conf --host localhost --port 58070 $(1) $(2) || (cat regression.diffs && false)
 endef
 
 define isolation_test
-	$(top_builddir)/src/test/isolation/pg_isolation_regress --dlpath=$(BASEPATH) $(EXTENSIONLOAD) --temp-instance ./tmp --temp-config ./isolation_postgresql.conf --host localhost --port 58071 $(1) $(2) || (cat regression.diffs && false)
+	$(top_builddir)/src/test/isolation/pg_isolation_regress --encoding=UTF8 --dlpath=$(BASEPATH) $(EXTENSIONLOAD) --temp-instance ./tmp --temp-config ./isolation_postgresql.conf --host localhost --port 58071 $(1) $(2) || (cat regression.diffs && false)
 endef
 
+check-basic: generate_version_schedule
+	$(call common_test,--schedule=./basic_schedule --schedule=./log/basic_schedule_$(pg_major_version))
+
 check-dist-minimal:
 	$(call common_test,--schedule=./minimal_schedule, $(EXTRA_TESTS))
 
-check-basic:
-	$(call common_test,--schedule=./basic_schedule --schedule=./basic_schedule_core)
-
 check-test-output:
-	./validate_test_output.sh
+	./validate_test_output.sh $(pg_major_version)
+
+generate_version_schedule:
+	@mkdir -p log
+	@cp basic_schedule_core log/basic_schedule_$(pg_major_version)
+	@./mutate_schedule.sh log/basic_schedule_$(pg_major_version) $(pg_major_version)
+
 
 all: check-basic check-test-output
@@ -37,7 +37,7 @@ test: bson_dollar_operators_negation_tests_runtime
 test: bson_dollar_operators_negation_tests_index
 test: bson_dollar_operators_negation_tests_explain_index
 test: bson_dollar_operators_negation_tests_explain_runtime
-test: bson_dollar_ops_collation_runtime_tests 
+test: bson_dollar_ops_collation_tests_runtime 
 
 test: create_indexes_cosmos_vector
 test: bson_query_operator_geospatial_tests_index

@@ -3144,7 +3144,7 @@ select * from bson_dollar_project('{}','{"result": {"$dateFromString": {"dateStr
 select * from bson_dollar_project('{}','{"result": {"$dateFromString": {"dateString": "20-1-1", "format": "%Y-%m-%d"} } }');
                         bson_dollar_project                         
 ---------------------------------------------------------------------
- { "result" : { "$date" : { "$numberLong" : "-61536038400000" } } }
+ { "result" : { "$date" : { "$numberLong" : "-61536038822000" } } }
 (1 row)
 
 select * from bson_dollar_project('{}','{"result": {"$dateFromString": {"dateString": "Year 2025, Month 12, Day 31", "format": "Year %Y, Month %m, Day %d"} } }');
@@ -3183,7 +3183,7 @@ select * from bson_dollar_project('{}','{"result": {"$dateFromString": {"dateStr
 select * from bson_dollar_project('{}','{"result": {"$dateFromString": {"dateString": "1.1.1", "format": "%V.%u.%G"} } }');
                         bson_dollar_project                         
 ---------------------------------------------------------------------
- { "result" : { "$date" : { "$numberLong" : "-62135568000000" } } }
+ { "result" : { "$date" : { "$numberLong" : "-62135568422000" } } }
 (1 row)
 
 -- iso part only requires year
@@ -3228,7 +3228,7 @@ select * from bson_dollar_project('{"dateString": "2025-12-31", "format": "%Y-%m
 select * from bson_dollar_project('{"dateString": "20-1-1", "format": "%Y-%m-%d"}','{"result": {"$dateFromString": {"dateString": "$dateString" , "format": "$format"} } }');
                         bson_dollar_project                         
 ---------------------------------------------------------------------
- { "result" : { "$date" : { "$numberLong" : "-61536038400000" } } }
+ { "result" : { "$date" : { "$numberLong" : "-61536038822000" } } }
 (1 row)
 
 select * from bson_dollar_project('{"dateString": "Year 2025, Month 12, Day 31", "format": "Year %Y, Month %m, Day %d"}','{"result": {"$dateFromString": {"dateString": "$dateString" , "format": "$format"} } }');
@@ -3267,7 +3267,7 @@ select * from bson_dollar_project('{"dateString": "53.7.2017", "format": "%V.%u.
 select * from bson_dollar_project('{"dateString": "1.1.1", "format": "%V.%u.%G"}','{"result": {"$dateFromString": {"dateString": "$dateString" , "format": "$format"} } }');
                         bson_dollar_project                         
 ---------------------------------------------------------------------
- { "result" : { "$date" : { "$numberLong" : "-62135568000000" } } }
+ { "result" : { "$date" : { "$numberLong" : "-62135568422000" } } }
 (1 row)
 
 -- iso part only requires year