infra: use spark connect to run pytests (#2491) · bookingcom/iceberg-python@513295d · GitHub
Skip to content

Commit 513295d

Browse files
kevinjqliuFokko
kevinjqliu
and
Fokko
authored
infra: use spark connect to run pytests (apache#2491)
<!-- Thanks for opening a pull request! --> <!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. --> <!-- Closes #${GITHUB_ISSUE_ID} --> # Rationale for this change Closes apache#2492 Run pytest using Spark Connect for a more consistent test env + a few general cleanup changes ## Are these changes tested? ## Are there any user-facing changes? <!-- In the case of user-facing changes, please add the changelog label. --> --------- Co-authored-by: Fokko Driesprong <fokko@apache.org>
1 parent e3e0ec6 commit 513295d

13 files changed

Lines changed: 153 additions & 255 deletions

‎.gitignore‎

Lines changed: 0 additions & 7 deletions

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# Configuration Variables
1919
# ========================
2020

21-
PYTEST_ARGS ?= -v # Override with e.g. PYTEST_ARGS="-vv --tb=short"
21+
PYTEST_ARGS ?= -v -x # Override with e.g. PYTEST_ARGS="-vv --tb=short"
2222
COVERAGE ?= 0 # Set COVERAGE=1 to enable coverage: make test COVERAGE=1
2323
COVERAGE_FAIL_UNDER ?= 85 # Minimum coverage % to pass: make coverage-report COVERAGE_FAIL_UNDER=70
2424
KEEP_COMPOSE ?= 0 # Set KEEP_COMPOSE=1 to keep containers after integration tests
@@ -37,7 +37,7 @@ endif
3737
ifeq ($(KEEP_COMPOSE),1)
3838
CLEANUP_COMMAND = echo "Keeping containers running for debugging (KEEP_COMPOSE=1)"
3939
else
40-
CLEANUP_COMMAND = docker compose -f dev/docker-compose-integration.yml down -v --remove-orphans 2>/dev/null || true
40+
CLEANUP_COMMAND = docker compose -f dev/docker-compose-integration.yml down -v --remove-orphans --timeout 0 2>/dev/null || true
4141
endif
4242

4343
# ============

‎dev/Dockerfile‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,13 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
3636
RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
3737
WORKDIR ${SPARK_HOME}
3838

39-
# Remember to also update `tests/conftest`'s spark setting
4039
ENV SPARK_VERSION=3.5.6
41-
ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
42-
ENV ICEBERG_VERSION=1.9.1
40+
ENV SCALA_VERSION=2.12
41+
ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_${SCALA_VERSION}
42+
ENV ICEBERG_VERSION=1.9.2
4343
ENV PYICEBERG_VERSION=0.10.0
44+
ENV HADOOP_VERSION=3.3.4
45+
ENV AWS_SDK_VERSION=1.12.753
4446

4547
# Try the primary Apache mirror (downloads.apache.org) first, then fall back to the archive
4648
RUN set -eux; \
@@ -59,15 +61,26 @@ RUN set -eux; \
5961
tar xzf "$FILE" --directory /opt/spark --strip-components 1; \
6062
rm -rf "$FILE"
6163

64+
# Download Spark Connect server JAR
65+
RUN curl --retry 5 -s -L https://repo1.maven.org/maven2/org/apache/spark/spark-connect_${SCALA_VERSION}/${SPARK_VERSION}/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar \
66+
-Lo /opt/spark/jars/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar
67+
6268
# Download iceberg spark runtime
6369
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
6470
-Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
6571

66-
6772
# Download AWS bundle
6873
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
6974
-Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
7075

76+
# Download hadoop-aws (required for S3 support)
77+
RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
78+
-Lo /opt/spark/jars/hadoop-aws-${HADOOP_VERSION}.jar
79+
80+
# Download AWS SDK bundle
81+
RUN curl --retry 5 -s https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar \
82+
-Lo /opt/spark/jars/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar
83+
7184
COPY spark-defaults.conf /opt/spark/conf
7285
ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"
7386

‎dev/docker-compose-integration.yml‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,13 @@ services:
2626
- rest
2727
- hive
2828
- minio
29-
volumes:
30-
- ./warehouse:/home/iceberg/warehouse
3129
environment:
3230
- AWS_ACCESS_KEY_ID=admin
3331
- AWS_SECRET_ACCESS_KEY=password
3432
- AWS_REGION=us-east-1
3533
ports:
36-
- 8888:8888
37-
- 8080:8080
34+
- 15002:15002 # Spark Connect
35+
- 4040:4040 # Spark UI
3836
links:
3937
- rest:rest
4038
- hive:hive

‎dev/entrypoint.sh‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@
1818
# under the License.
1919
#
2020

21-
start-master.sh -p 7077
22-
start-worker.sh spark://spark-iceberg:7077
23-
start-history-server.sh
21+
start-connect-server.sh
2422

2523
tail -f /dev/null

‎dev/provision.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@
5050
"hive",
5151
**{
5252
"type": "hive",
53-
"uri": "http://hive:9083",
53+
"uri": "thrift://hive:9083",
5454
"s3.endpoint": "http://minio:9000",
5555
"s3.access-key-id": "admin",
5656
"s3.secret-access-key": "password",

‎dev/spark-defaults.conf‎

Lines changed: 18 additions & 3 deletions

0 commit comments

Comments
 (0)