RedisGraph · AviAvni · Feb 20, 2023 · Jan 17, 2023 · Jan 18, 2023 · Jan 18, 2023
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -64,10 +64,19 @@ commands:
           name: Setup automation
           command: |
             git submodule update --init deps/readies
+            if [[ $(uname -s) == Darwin ]]; then rm -f /usr/local/bin/python3; fi
             ./deps/readies/bin/getpy3
+      - run:
+          name: Setup automation (2)
+          shell: /bin/bash -l -eo pipefail
+          command: |
+            export HOMEBREW_NO_AUTO_UPDATE=1
             ./deps/readies/bin/getpy2
             ./deps/readies/bin/getaws
-          no_output_timeout: 30m
+            ls -l /usr/local/bin/python* || true
+            echo "python3: $(command -v python3)"
+            python3 --version
+            python3 -m pip list
 
   install-prerequisites:
     parameters:

diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml
@@ -0,0 +1,37 @@
+name: Build on macOS
+
+on:
+  push:
+  workflow_dispatch:
+
+jobs:
+  build-macos:
+    runs-on: macos-11
+    defaults:
+      run:
+        shell: bash -l -eo pipefail {0}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: Setup
+      run: |
+        ./sbin/setup
+        export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}"
+        ./deps/readies/bin/getredis --with-github-token
+    - name: Build
+      run: |
+        make
+        make pack
+    - name: Unit tests
+      run: |
+        make unit-tests CLEAR_LOGS=0
+    - name: Flow tests
+      run: |
+        make flow-tests CLEAR_LOGS=0
+    - name: TCK tests
+      run: |
+        make tck-tests CLEAR_LOGS=0
+    - name: Fuzzer tests
+      run: |
+        make fuzz
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,15 +9,11 @@ endif()
 
 project(redisgraph)
 
-file(GLOB_RECURSE SOURCES "src/*.c")
+setup_cc_options()
 
-if (NOT APPLE)
-	add_library(redisgraph SHARED ${SOURCES})
-else()
-	add_library(redisgraph MODULE ${SOURCES})
-endif()
+file(GLOB_RECURSE SOURCES "src/*.c")
 
-set_property(TARGET redisgraph PROPERTY C_STANDARD 11)
+add_library(redisgraph SHARED ${SOURCES})
 
 include_directories(
 	${root}/src
@@ -41,17 +37,11 @@ if (DEBUG)
 	add_compile_definitions(RG_DEBUG)
 endif()
 
-add_compile_options(${CMAKE_CC_FLAGS} ${CMAKE_CC_C_FLAGS})
-target_compile_options(redisgraph PRIVATE -fPIC ${CMAKE_CC_FLAGS} ${CMAKE_CC_C_FLAGS})
 if (MEMCHECK)
 	add_compile_definitions(MEMCHECK)
 endif()
 
-lists_from_env(LIBOMP)
-target_link_options(redisgraph PRIVATE ${CMAKE_LD_FLAGS} ${CMAKE_SO_LD_FLAGS} ${LIBOMP})
-
-set_target_properties(redisgraph PROPERTIES PREFIX "")
-set_target_properties(redisgraph PROPERTIES SUFFIX ".so")
+setup_shared_object_target(redisgraph "")
 
 set_target_properties(redisgraph PROPERTIES LINKER_LANGUAGE CXX)
 
@@ -60,17 +50,16 @@ set(REDISGRAPH_OBJECTS $<TARGET_OBJECTS:redisgraph>)
 lists_from_env(GRAPHBLAS LIBXXHASH RAX LIBCYPHER_PARSER REDISEARCH_LIBS UTF8PROC)
 set(REDISGRAPH_LIBS ${GRAPHBLAS} ${LIBXXHASH} ${RAX} ${LIBCYPHER_PARSER} ${REDISEARCH_LIBS} ${UTF8PROC})
 
-target_link_libraries(redisgraph PRIVATE ${REDISGRAPH_LIBS} c m dl pthread)
+target_link_options(redisgraph PRIVATE ${CMAKE_LD_FLAGS_LIST} ${CMAKE_SO_LD_FLAGS_LIST})
+target_link_libraries(redisgraph PRIVATE ${REDISGRAPH_LIBS} ${CMAKE_LD_LIBS})
 
 extract_debug_symbols(redisgraph)
 
 if (APPLE)
-	# add_library(redisgraph_dylib SHARED $<TARGET_OBJECTS:redisgraph>)
-	# target_link_libraries(redisgraph_dylib PRIVATE ${REDISGRAPH_LIBS} ${LIBOMP} c m dl pthread)
-
 	add_library(redisgraph_static STATIC $<TARGET_OBJECTS:redisgraph>)
 endif()
 
 if (UNIT_TESTS)
 	add_subdirectory(${root}/tests/unit tests/unit)
 endif()
+
diff --git a/Makefile b/Makefile
@@ -141,6 +141,9 @@ LIBS=$(RAX) $(LIBXXHASH) $(GRAPHBLAS) $(REDISEARCH_LIBS) $(LIBCYPHER_PARSER) $(U
 
 CC_COMMON_H=$(SRCDIR)/src/common.h
 
+CC_C_STD=gnu11
+CC_OPENMP=1
+
 include $(MK)/defs
 
 $(info # Building into $(BINDIR))
@@ -296,7 +299,7 @@ endif
 
 #----------------------------------------------------------------------------------------------
 
-pack package: $(TARGET)
+pack package: #$(TARGET)
 	@MODULE=$(realpath $(TARGET)) $(ROOT)/sbin/pack.sh
 
 upload-release:
@@ -321,14 +324,7 @@ ifneq ($(BUILD),0)
 TEST_DEPS=$(TARGET)
 endif
 
-ifeq ($(VG_DOCKER),1)
-test:
-	@echo Building docker to run valgrind on macOS
-	$(SHOW)docker build -f tests/Dockerfile -t macos_test_docker .
-else
-test: $(TEST_DEPS)
-	$(SHOW)MODULE=$(TARGET) BINROOT=$(BINROOT) PARALLEL=$(_RLTEST_PARALLEL) ./tests/flow/tests.sh
-endif
+test: unit-tests flow-tests tck-tests
 
 unit-tests:
 ifneq ($(BUILD),0)
@@ -385,7 +381,9 @@ COV_EXCLUDE+=$(foreach D,$(COV_EXCLUDE_DIRS),'$(realpath $(ROOT))/$(D)/*')
 coverage:
 	$(SHOW)$(MAKE) build COV=1
 	$(SHOW)$(COVERAGE_RESET)
-	-$(SHOW)$(MAKE) test COV=1
+	-$(SHOW)$(MAKE) unit-tests COV=1
+	-$(SHOW)$(MAKE) flow-tests COV=1
+	-$(SHOW)$(MAKE) tck-tests COV=1
 	$(SHOW)$(COVERAGE_COLLECT_REPORT)
 
 .PHONY: coverage

diff --git a/build/docker/dockerfile.tmpl b/build/docker/dockerfile.tmpl
@@ -19,7 +19,7 @@ ADD . /build
 RUN ./deps/readies/bin/getbash
 {% endif %}
 RUN ./deps/readies/bin/getupdates
-RUN ./sbin/setup
+RUN VERBOSE=1 ./sbin/setup
 
 RUN make
 

diff --git a/deps/readies b/deps/readies
diff --git a/sbin/pack.sh b/sbin/pack.sh
@@ -37,13 +37,17 @@ if [[ $1 == --help || $1 == help || $HELP == 1 ]]; then
 		JUST_PRINT=1        Only print package names, do not generate
 		VERBOSE=1           Print commands
 		IGNERR=1            Do not abort on error
+		NOP=1               Print commands, do not execute
 
 	END
 	exit 0
 fi
 
 #----------------------------------------------------------------------------------------------
 
+OP=""
+[[ $NOP == 1 ]] && OP=echo
+
 ARCH=$($READIES/bin/platform --arch)
 [[ $ARCH == x64 ]] && ARCH=x86_64
 
@@ -83,7 +87,7 @@ fi
 mkdir -p $ARTDIR $ARTDIR/snapshots
 ARTDIR=$(cd $ARTDIR && pwd)
 
-MODULE_NAME=${MODULE_NAME:-redisgraph}
+MODULE_NAME=${MODULE_NAME:-graph}
 PACKAGE_NAME=${PACKAGE_NAME:-redisgraph}
 
 DEP_NAMES=""
@@ -135,45 +139,57 @@ pack_ramp() {
 		$xtx_vars \
 		-e NUMVER -e SEMVER \
 		$RAMP_YAML > /tmp/ramp.yml
-
-	rm -f /tmp/ramp.fname $packfile
-	$RAMP_CMD pack -m /tmp/ramp.yml \
-		$RAMP_ARGS \
-		-n $MODULE_NAME \
-		--verbose \
-		--debug \
-		--packname-file /tmp/ramp.fname \
-		-o $packfile \
-		$MODULE \
-		>/tmp/ramp.err 2>&1 || true
-
-	if [[ ! -e $packfile ]]; then
-		eprint "Error generating RAMP file:"
-		>&2 cat /tmp/ramp.err
-		exit 1
-	else
-		local packname=`cat /tmp/ramp.fname`
-		echo "Created $packname"
+	if [[ $VERBOSE == 1 ]]; then
+		echo "# ramp.yml:"
+		cat /tmp/ramp.yml
 	fi
 
-	if [[ -f $MODULE.debug ]]; then
+	runn rm -f /tmp/ramp.fname $packfile
+	runn @ <<-EOF
 		$RAMP_CMD pack -m /tmp/ramp.yml \
 			$RAMP_ARGS \
 			-n $MODULE_NAME \
 			--verbose \
 			--debug \
 			--packname-file /tmp/ramp.fname \
-			-o $packfile_debug \
-			$MODULE.debug \
+			-o $packfile \
+			$MODULE \
 			>/tmp/ramp.err 2>&1 || true
+		EOF
 
-		if [[ ! -e $packfile_debug ]]; then
+	if [[ $NOP != 1 ]]; then
+		if [[ ! -e $packfile ]]; then
 			eprint "Error generating RAMP file:"
 			>&2 cat /tmp/ramp.err
 			exit 1
 		else
 			local packname=`cat /tmp/ramp.fname`
-			echo "Created $packname"
+			echo "# Created $packname"
+		fi
+	fi
+
+	if [[ -f $MODULE.debug ]]; then
+		runn @ <<-EOF
+			$RAMP_CMD pack -m /tmp/ramp.yml \
+				$RAMP_ARGS \
+				-n $MODULE_NAME \
+				--verbose \
+				--debug \
+				--packname-file /tmp/ramp.fname \
+				-o $packfile_debug \
+				$MODULE.debug \
+				>/tmp/ramp.err 2>&1 || true
+			EOF
+
+		if [[ $NOP != 1 ]]; then
+			if [[ ! -e $packfile_debug ]]; then
+				eprint "Error generating RAMP file:"
+				>&2 cat /tmp/ramp.err
+				exit 1
+			else
+				local packname=`cat /tmp/ramp.fname`
+				echo "# Created $packname"
+			fi
 		fi
 	fi
 
@@ -196,37 +212,49 @@ pack_deps() {
 	local dep_prefix_dir=$(cat $ARTDIR/$dep.prefix)
 
 	rm -f $tar_path
-	{ cd $depdir ;\
-	  cat $ARTDIR/$dep.files | \
-	  xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' \
-	  	--transform "s,^,$dep_prefix_dir," 2>> /tmp/pack.err | \
-	  gzip -n - > $tar_path ; E=$?; } || true
-	if [[ ! -e $tar_path || -z $(tar tzf $tar_path) ]]; then
-		eprint "Count not create $tar_path. Aborting."
-		rm -f $tar_path
-		exit 1
+	if [[ $NOP != 1 ]]; then
+		{ cd $depdir ;\
+		  cat $ARTDIR/$dep.files | \
+		  xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' \
+			--transform "s,^,$dep_prefix_dir," 2>> /tmp/pack.err | \
+		  gzip -n - > $tar_path ; E=$?; } || true
+		if [[ ! -e $tar_path || -z $(tar tzf $tar_path) ]]; then
+			eprint "Count not create $tar_path. Aborting."
+			rm -f $tar_path
+			exit 1
+		fi
+	else
+		runn @ <<-EOF
+			cd $depdir
+			cat $ARTDIR/$dep.files | \
+			xargs tar -c --sort=name --owner=root:0 --group=root:0 --mtime='UTC 1970-01-01' \
+				--transform "s,^,$dep_prefix_dir," 2>> /tmp/pack.err | \
+			gzip -n - > $tar_path ; E=$?; } || true
+			EOF
 	fi
-	sha256sum $tar_path | awk '{print $1}' > $tar_path.sha256
+	runn @ <<-EOF
+		sha256sum $tar_path | awk '{print $1}' > $tar_path.sha256
+		EOF
 
 	mkdir -p $ARTDIR/snapshots
 	cd $ARTDIR/snapshots
 	if [[ ! -z $BRANCH ]]; then
 		local snap_package=$stem.${BRANCH}${VARIANT}.tgz
-		ln -sf ../$fq_package $snap_package
-		ln -sf ../$fq_package.sha256 $snap_package.sha256
+		runn ln -sf ../$fq_package $snap_package
+		runn ln -sf ../$fq_package.sha256 $snap_package.sha256
 	fi
 }
 
 #----------------------------------------------------------------------------------------------
 
 prepare_symbols_dep() {
 	if [[ ! -f $MODULE.debug ]]; then return 0; fi
-	echo "Preparing debug symbols dependencies ..."
+	echo "# Preparing debug symbols dependencies ..."
 	echo $(dirname $(realpath $MODULE)) > $ARTDIR/debug.dir
 	echo $(basename $(realpath $MODULE)).debug > $ARTDIR/debug.files
 	echo "" > $ARTDIR/debug.prefix
 	pack_deps debug
-	echo "Done."
+	echo "# Done."
 }
 
 #----------------------------------------------------------------------------------------------
@@ -282,15 +310,15 @@ fi
 #----------------------------------------------------------------------------------------------
 
 if [[ $DEPS == 1 ]]; then
-	echo "Building dependencies ..."
+	echo "# Building dependencies ..."
 
 	[[ $SYM == 1 ]] && prepare_symbols_dep
 
 	for dep in $DEP_NAMES; do
-		echo "$dep ..."
+		echo "# $dep ..."
 		pack_deps $dep
 	done
-	echo "Done."
+	echo "# Done."
 fi
 
 #----------------------------------------------------------------------------------------------
@@ -303,7 +331,7 @@ if [[ $RAMP == 1 ]]; then
 		exit 1
 	fi
 
-	echo "Building RAMP files ..."
+	echo "# Building RAMP files ..."
 
 	[[ -z $MODULE ]] && { eprint "Nothing to pack. Aborting."; exit 1; }
 	[[ ! -f $MODULE ]] && { eprint "$MODULE does not exist. Aborting."; exit 1; }
@@ -312,12 +340,12 @@ if [[ $RAMP == 1 ]]; then
 	[[ $RELEASE == 1 ]] && SNAPSHOT=0 pack_ramp
 	[[ $SNAPSHOT == 1 ]] && pack_ramp
 
-	echo "Done."
+	echo "# Done."
 fi
 
 if [[ $VERBOSE == 1 ]]; then
-	echo "Artifacts:"
-	du -ah --apparent-size $ARTDIR
+	echo "# Artifacts:"
+	$OP du -ah --apparent-size $ARTDIR
 fi
 
 exit 0
+1 −0		bin/dockerama
+3 −1		bin/getgit
+67 −8		bin/getpy3
+7 −3		bin/getpyenv
+11 −3		bin/getredis
+8 −0		bin/getrust
+18 −19		cmake/cc
+16 −1		cmake/functions
+37 −13		mk/cc.defs
+1 −1		mk/main
+11 −5		mk/openmp.defs
+1 −1		paella/setup.py
+6 −0		shibumi/functions