iframe-proxy | Sunbelt Computer Software

Repository navigation

History

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241

242

243

244

245

246

247

248

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

357

358

359

360

361

362

363

364

365

366

367

368

369

370

371

372

373

374

375

376

377

378

379

380

381

382

383

384

385

386

387

388

389

390

391

392

393

394

395

396

397

398

399

400

401

402

403

404

405

406

407

408

409

410

411

412

413

414

415

416

417

418

419

420

421

422

423

424

425

426

427

428

429

430

431

432

433

434

435

436

437

438

439

440

441

442

443

444

445

446

447

448

449

450

451

452

453

454

455

456

457

458

459

460

461

462

463

464

465

466

467

468

469

470

471

472

473

474

475

476

477

478

479

480

481

482

483

484

485

486

487

488

489

490

491

492

493

494

495

496

497

498

499

500

501

502

503

504

#!/bin/bash

# Licensed to the Apache Software Foundation (ASF) under one

# or more contributor license agreements. See the NOTICE file

# distributed with this work for additional information

# regarding copyright ownership. The ASF licenses this file

# to you under the Apache License, Version 2.0 (the

# "License"); you may not use this file except in compliance

# with the License. You may obtain a copy of the License at

# http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing,

# software distributed under the License is distributed on an

# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY

# KIND, either express or implied. See the License for the

# specific language governing permissions and limitations

# under the License.

# This script bootstraps a system for Impala development from almost nothing; it is known

# to work on Ubuntu 20.04. It clobbers some local environment and system

# configurations, so it is best to run this in a fresh install. It also sets up the

# ~/.bashrc for the calling user and impala-config-local.sh with some environment

# variables to make Impala compile and run after this script is complete.

# When IMPALA_HOME is set, the script will bootstrap Impala development in the

# location specified.

# The intended user is a person who wants to start contributing code to Impala. This

# script serves as an executable reference point for how to get started.

# To run this in a Docker container:

# 1. Run with --privileged

# 2. Give the container a non-root sudoer wih NOPASSWD:

# apt-get update

# apt-get install sudo

# adduser --disabled-password --gecos '' impdev

# echo 'impdev ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers

# 3. Run this script as that user: su - impdev -c /bootstrap_development.sh

set -eu -o pipefail

: ${IMPALA_HOME:=$(cd "$(dirname $0)"/..; pwd)}

export IMPALA_HOME

if [[ -t 1 ]] # if on an interactive terminal

then

echo "This script will clobber some system settings. Are you sure you want to"

echo -n "continue? "

while true

read -p "[yes/no] " ANSWER

ANSWER=$(echo "$ANSWER" | tr /a-z/ /A-Z/)

if [[ $ANSWER = YES ]]

then

break

elif [[ $ANSWER = NO ]]

then

echo "OK, Bye!"

exit 1

done

set -x

# Determine whether we're running on redhat or ubuntu

REDHAT=

REDHAT8=

REDHAT9=

REDHAT10=

UBUNTU=

UBUNTU20=

UBUNTU22=

UBUNTU24=

IN_DOCKER=${IN_DOCKER:-}

DOCKER_BUILD=${DOCKER_BUILD:-}

if [[ -f /etc/redhat-release ]]; then

REDHAT=true

echo "Identified redhat system."

if grep 'release 10\.' /etc/redhat-release; then

REDHAT10=true

echo "Identified redhat10 system."

elif grep 'release 9\.' /etc/redhat-release; then

REDHAT9=true

echo "Identified redhat9 system."

elif grep 'release 8\.' /etc/redhat-release; then

REDHAT8=true

echo "Identified redhat8 system."

else

echo "This script only supports Redhat releases 8, 9, or 10 (or equivalents)."

else

source /etc/lsb-release

if [[ $DISTRIB_ID = Ubuntu ]]

then

UBUNTU=true

echo "Identified Ubuntu system."

# Kerberos setup would pop up dialog boxes without this

export DEBIAN_FRONTEND=noninteractive

if [[ $DISTRIB_RELEASE = 20.04 ]]

then

UBUNTU20=true

echo "Identified Ubuntu 20.04 system."

elif [[ $DISTRIB_RELEASE = 22.04 ]]

then

UBUNTU22=true

echo "Identified Ubuntu 22.04 system."

elif [[ $DISTRIB_RELEASE = 24.04 ]]

then

UBUNTU24=true

echo "Identified Ubuntu 24.04 system."

else

echo "This script supports Ubuntu versions 20.04, 22.04, or 24.04" >&2

exit 1

else

echo "This script only supports Ubuntu or RedHat" >&2

exit 1

if [[ "${IN_DOCKER}" == true ]] || grep docker /proc/1/cgroup; then

IN_DOCKER=true

echo "Identified we are running inside of Docker."

# Helper function to execute following command only on Ubuntu

function ubuntu {

if [[ "$UBUNTU" == true ]]; then

"$@"

}

function ubuntu20 {

if [[ "$UBUNTU20" == true ]]; then

"$@"

}

function ubuntu22 {

if [[ "$UBUNTU22" == true ]]; then

"$@"

}

function ubuntu24 {

if [[ "$UBUNTU24" == true ]]; then

"$@"

}

# Helper function to execute following command only on RedHat

function redhat {

if [[ "$REDHAT" == true ]]; then

"$@"

}

# Helper function to execute following command only on RedHat8

function redhat8 {

if [[ "$REDHAT8" == true ]]; then

"$@"

}

# Helper function to execute following command only on RedHat9

function redhat9 {

if [[ "$REDHAT9" == true ]]; then

"$@"

}

# Helper function to execute following command only on RedHat10

function redhat10 {

if [[ "$REDHAT10" == true ]]; then

"$@"

}

# Helper function to execute following command only in docker

function indocker {

if [[ "$IN_DOCKER" == true ]]; then

"$@"

}

# Helper function to execute following command only outside of docker

function notindocker {

if [[ "$IN_DOCKER" != true ]]; then

"$@"

}

ARCH_NAME=$(uname -m)

# X permission on home directory is needed for some uses of postgresql (IMPALA-13693)

chmod o+X ~

# Note that yum has its own retries; see yum.conf(5).

REAL_APT_GET=$(ubuntu which apt-get)

function apt-get {

for ITER in $(seq 1 30); do

echo "ATTEMPT: ${ITER}"

if sudo -E "${REAL_APT_GET}" "$@"

then

return 0

sleep "${ITER}"

done

echo "NO MORE RETRIES"

return 1

}

echo ">>> Installing build tools"

if [[ "$UBUNTU" == true ]]; then

while sudo fuser /var/lib/dpkg/lock-frontend; do

sleep 1

done

# Set UBUNTU_JAVA_VERSION, UBUNTU_PACKAGE_ARCH, REDHAT_JAVA_VERSION

source "$IMPALA_HOME/bin/impala-config-java.sh"

ubuntu apt-get update

ubuntu apt-get --yes install ccache curl file gawk g++ gcc apt-utils git libffi-dev \

libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \

libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \

python3-dev python3-setuptools python3-venv postgresql \

ssh wget vim-common psmisc lsof net-tools language-pack-en libxml2-dev \

libxslt-dev openjdk-${UBUNTU_JAVA_VERSION}-jdk \

openjdk-${UBUNTU_JAVA_VERSION}-source openjdk-${UBUNTU_JAVA_VERSION}-dbg \

xsltproc

# Required by Kudu in the minicluster. Older Kudu versions depend on libtinfo5,

# versions that can be compiled for Ubuntu 24.04 depend on libtinfo6.

ubuntu20 apt-get --yes install libtinfo5 libtinfo6

ubuntu22 apt-get --yes install libtinfo5 libtinfo6

ubuntu24 apt-get --yes install libtinfo6

ubuntu sudo update-java-alternatives -l || true

# Configure the default Java version to be the version we selected.

ubuntu sudo update-java-alternatives -v -s \

java-1.${UBUNTU_JAVA_VERSION}.0-openjdk-${UBUNTU_PACKAGE_ARCH}

redhat sudo yum install -y file gawk gcc gcc-c++ git krb5-devel krb5-server \

krb5-workstation libevent-devel libffi-devel make openssl-devel cyrus-sasl \

cyrus-sasl-gssapi cyrus-sasl-devel cyrus-sasl-plain \

postgresql postgresql-server rpm-build \

wget vim-common cmake zlib-devel \

procps psmisc lsof openssh-server \

net-tools langpacks-en glibc-langpack-en libxml2-devel libxslt-devel \

java-${REDHAT_JAVA_VERSION}-openjdk-src java-${REDHAT_JAVA_VERSION}-openjdk-devel

# NSCD was removed in redhat10

redhat8 sudo yum install -y nscd

redhat9 sudo yum install -y nscd

# We need Python >= 3.8, so Redhat 8 needs to manually specify a newer python

redhat8 sudo yum install -y python38-devel python38-setuptools

redhat9 sudo yum install -y python3-devel python3-setuptools

redhat10 sudo yum install -y python3-devel python3-setuptools

if [[ "$REDHAT10" == true ]] ; then

# Redhat 10 doesn't automatically create the alternatives family that we rely on

# below. Instead, this uses the paths directly.

sudo alternatives --set java /usr/lib/jvm/java-${REDHAT_JAVA_VERSION}-openjdk/bin/java

sudo alternatives --set javac \

/usr/lib/jvm/java-${REDHAT_JAVA_VERSION}-openjdk/bin/javac

sudo alternatives --set java_sdk_openjdk \

/usr/lib/jvm/java-${REDHAT_JAVA_VERSION}-openjdk

sudo alternatives --set jre_openjdk /usr/lib/jvm/java-${REDHAT_JAVA_VERSION}-openjdk

elif [[ "$REDHAT" == true ]] ; then

# Redhat 8 and 9 have alternatives family settings (i.e. aliases that we can use

# rather than the path).

sudo alternatives --set java java-${REDHAT_JAVA_VERSION}-openjdk.${ARCH_NAME}

sudo alternatives --set javac java-${REDHAT_JAVA_VERSION}-openjdk.${ARCH_NAME}

sudo alternatives --set java_sdk_openjdk \

java-${REDHAT_JAVA_VERSION}-openjdk.${ARCH_NAME}

sudo alternatives --set jre_openjdk java-${REDHAT_JAVA_VERSION}-openjdk.${ARCH_NAME}

redhat8 sudo alternatives --set python3 /usr/bin/python3.8

# update-java-alternatives may not take effect if there is a Java in PATH

which java

java -version

which javac

javac -version

python3 --version

# fuse-devel doesn't exist for Redhat 9

redhat8 sudo yum install -y fuse-devel curl

# Redhat 9 and 10 can have curl-minimal preinstalled, which can conflict with curl.

# Adding --allowerasing allows curl to replace curl-minimal.

redhat9 sudo yum install -y --allowerasing curl

redhat10 sudo yum install -y --allowerasing curl

# CentOS repos don't contain ccache, so install from EPEL

redhat8 sudo yum install -y epel-release

redhat9 sudo yum install -y epel-release

# Redhat 10 needs CRB to be enabled to install epel-release

if [[ "$REDHAT10" == true ]] ; then

# RHEL has different steps from Rocky/Alma

if grep "Red Hat Enterprise Linux" /etc/redhat-release ; then

# On RHEL, CRB is done through the subscription manager, but some RHEL systems don't

# have repositories enabled. For now, just use the direct link.

sudo yum install -y \

https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm

else

sudo dnf -y install dnf-plugins-core

sudo dnf config-manager --enable crb

sudo dnf -y install epel-release

redhat sudo yum install -y ccache

# Clean up yum caches

redhat sudo yum clean all

# Download Maven since the packaged version is pretty old.

: ${IMPALA_TOOLCHAIN_HOST:=native-toolchain.s3.amazonaws.com}

MVN_VERSION="3.9.15"

if [ ! -d "/usr/local/apache-maven-${MVN_VERSION}" ]; then

sudo wget -nv \

"https://${IMPALA_TOOLCHAIN_HOST}/maven/apache-maven-${MVN_VERSION}-bin.tar.gz"

sha512sum -c - <<< "33d81e0ec785f0207e3e5e3ffb61863e1dca5784c15ac3fb5ff105f69cffbea484eb8d473ea60467a63f7b0570eef8622f2fed8eee96acbe668aa313391cddb3 apache-maven-${MVN_VERSION}-bin.tar.gz"

sudo tar -C /usr/local -xzf "apache-maven-${MVN_VERSION}-bin.tar.gz"

# Ensure that Impala's preferred version is installed locally,

# even if a previous version exists there.

sudo ln -s -f "/usr/local/apache-maven-${MVN_VERSION}/bin/mvn" "/usr/local/bin"

sudo rm "apache-maven-${MVN_VERSION}-bin.tar.gz"

# reset permissions on redhat

# TODO: figure out why this is necessary for redhat

MAVEN_DIRECTORY="/usr/local/apache-maven-${MVN_VERSION}"

redhat indocker sudo chmod 0755 ${MAVEN_DIRECTORY}

redhat indocker sudo chmod 0755 ${MAVEN_DIRECTORY}/{bin,boot}

if ! { service --status-all | grep -E '^ \[ \+ \] ssh$'; }

then

ubuntu sudo service ssh start

redhat notindocker sudo service sshd start

redhat indocker sudo /usr/bin/ssh-keygen -A

redhat indocker sudo /usr/sbin/sshd

# The CentOS 8.1 image includes /var/run/nologin by mistake; this file prevents

# SSH logins. See https://github.com/CentOS/sig-cloud-instance-images/issues/60

redhat8 indocker sudo rm -f /var/run/nologin

# TODO: config ccache to give it plenty of space

# TODO: check that there is enough space on disk to do a build and data load

# TODO: make this work with non-bash shells

echo ">>> Configuring system"

function setup_postgresql() {

echo ">>> Configuring postgresql. This can fail if postgres is already initialized"

# initdb can fail if it was run before on this host - ignore this error

redhat notindocker sudo service postgresql initdb || true

redhat notindocker sudo service postgresql stop

redhat indocker sudo -u postgres PGDATA=/var/lib/pgsql/data pg_ctl init

ubuntu sudo service postgresql stop

# These configurations expose connectiong to PostgreSQL via md5-hashed

# passwords over TCP to localhost, and the local socket is trusted

# widely.

ubuntu sudo sed -ri 's/local +all +all +peer/local all all trust/g' \

/etc/postgresql/*/main/pg_hba.conf

# Accept remote connections from the hosts in the same subnet.

ubuntu sudo sed -ri "s/#listen_addresses = 'localhost'/listen_addresses = '0.0.0.0'/g" \

/etc/postgresql/*/main/postgresql.conf

ubuntu sudo sed -ri 's/host +all +all +127.0.0.1\/32/host all all samenet/g' \

/etc/postgresql/*/main/pg_hba.conf

redhat sudo sed -ri 's/local +all +all +(ident|peer)/local all all trust/g' \

/var/lib/pgsql/data/pg_hba.conf

# Accept md5 passwords from localhost

redhat sudo sed -i -e 's,$host.*$ident,\1md5,' /var/lib/pgsql/data/pg_hba.conf

# Accept remote connections from the hosts in the same subnet.

redhat sudo sed -ri "s/#listen_addresses = 'localhost'/listen_addresses = '0.0.0.0'/g" \

/var/lib/pgsql/data/postgresql.conf

redhat sudo sed -ri 's/host +all +all +127.0.0.1\/32/host all all samenet/g' \

/var/lib/pgsql/data/pg_hba.conf

ubuntu sudo service postgresql start

redhat notindocker sudo service postgresql start

# Important to redirect pg_ctl to a logfile, lest it keep the stdout

# file descriptor open, preventing the shell from exiting.

redhat indocker sudo -u postgres PGDATA=/var/lib/pgsql/data bash -c \

"pg_ctl start -w --timeout=120 >> /var/lib/pgsql/pg.log 2>&1"

# Set up postgres for HMS

if ! [[ 1 = $(sudo -u postgres psql -At -c "SELECT count(*) FROM pg_roles WHERE rolname = 'hiveuser';") ]]

then

sudo -u postgres psql -c "CREATE ROLE hiveuser LOGIN PASSWORD 'password';"

sudo -u postgres psql -c "ALTER ROLE hiveuser WITH CREATEDB;"

# The output of "select * from pg_roles" may cause psql to block waiting for

# something to dismiss the terminal output. To avoid that, send output through

# tee.

sudo -u postgres psql -c "SELECT * FROM pg_roles WHERE rolname = 'hiveuser';" | tee

echo ">>> Configuring postgresql finished."

}

# Setup ssh to ssh to localhost

mkdir -p ~/.ssh

chmod go-rwx ~/.ssh

if ! [[ -f ~/.ssh/id_rsa ]]

then

ssh-keygen -t rsa -N '' -q -f ~/.ssh/id_rsa

{ echo "" | cat - ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys; } && chmod 0600 ~/.ssh/authorized_keys

echo -e "\nNoHostAuthenticationForLocalhost yes" >> ~/.ssh/config && chmod 0600 ~/.ssh/config

ssh localhost whoami

# Workarounds for HDFS networking issues: On the minicluster, tests that rely

# on WebHDFS may fail with "Connection refused" errors because the namenode

# will return a "Location:" redirect to the hostname, but the datanode is only

# listening on localhost. See also HDFS-13797. To reproduce this, the following

# snippet may be useful:

# $impala-python3

# >>> import logging

# >>> logging.basicConfig(level=logging.DEBUG)

# >>> logging.getLogger("requests.packages.urllib3").setLevel(logging.DEBUG)

# >>> from pywebhdfs.webhdfs import PyWebHdfsClient

# >>> PyWebHdfsClient(host='localhost',port='5070', user_name='hdfs').read_file(

# "/test-warehouse/tpch.region/region.tbl")

# INFO:...:Starting new HTTP connection (1): localhost

# DEBUG:...:"GET /webhdfs/v1//t....tbl?op=OPEN&user.name=hdfs HTTP/1.1" 307 0

# INFO:...:Starting new HTTP connection (1): HOSTNAME.DOMAIN

# Traceback (most recent call last):

# ...

# ...ConnectionError: ('Connection aborted.', error(111, 'Connection refused'))

# Prefer the FQDN first for rpc-mgr-kerberized-test as newer krb5 requires FQDN.

# During docker builds, the /etc/hosts file is read only and hosts are configured using

# the --add-host command line arg.

if [[ "${DOCKER_BUILD}" != "true" ]]; then

add_if_not_there() {

grep -q "$2" $1 || echo "$2" | sudo tee -a $1

}

# Container hostname is random during the build process and is different at run time.

# Thus, we should not add current hostname to /etc/hosts during the Docker build since

# that entry would also be in the runtime container.

add_if_not_there "/etc/hosts" "127.0.0.1 $(hostname) $(hostname -s)"

# Add hostnames with multiple labels to allow matching wildcard TLS certificates.

# Create names that map to v4/v6/dual localhost to help ipv6 testing.

add_if_not_there "/etc/hosts" "127.0.0.1 ip4.impala.test ip46.impala.test"

add_if_not_there "/etc/hosts" "::1 ip6.impala.test ip46.impala.test"

# In Docker, one can change /etc/hosts as above but not with sed -i. The error message

# is "sed: cannot rename /etc/sedc3gPj8: Device or resource busy". The following lines

# are basically sed -i but with cp instead of mv for -i part.

if [[ $(grep -c 127.0.1.1 /etc/hosts || true) != "0" ]]; then

NEW_HOSTS=$(mktemp)

sed 's/127.0.1.1/127.0.0.1/g' /etc/hosts > "${NEW_HOSTS}"

diff -u /etc/hosts "${NEW_HOSTS}" || true

sudo cp "${NEW_HOSTS}" /etc/hosts

rm "${NEW_HOSTS}"

sudo mkdir -p /var/lib/hadoop-hdfs

sudo chown $(whoami) /var/lib/hadoop-hdfs/

# TODO: restrict this to only the users it is needed for

echo -e "\n* - nofile 1048576" | sudo tee -a /etc/security/limits.conf

# Increase memlock for HDFS caching. On RedHat systems this defaults to 64 (KB). Ubuntu

# uses systemd, which sets its own default. With Ubuntu 18.04 that default is 16 KB,

# 20.04+ defaults to 64 MB. Set all to 64 MB for the current user; Impala test systems

# require 10s of GBs of memory, so this setting should not be a problem.

USER=${USER-$(id -un)}

echo -e "${USER} - memlock 65536" | sudo tee /etc/security/limits.d/10-memlock.conf

# Default on CentOS limits a user to 1024 or 4096 processes (threads) , which isn't

# enough for minicluster with all of its friends.

redhat echo -e "* soft nproc unlimited" | sudo tee -a /etc/security/limits.conf

echo ">>> Checking out Impala"

# If there is no Impala git repo, get one now

if ! [[ -d "$IMPALA_HOME" ]]

then

time -p git clone https://gitbox.apache.org/repos/asf/impala.git "$IMPALA_HOME"

cd "$IMPALA_HOME"

SET_IMPALA_HOME="export IMPALA_HOME=$(pwd)"

echo -e "\n$SET_IMPALA_HOME" >> ~/.bashrc

eval "$SET_IMPALA_HOME"

# Optionally try to prepopulate the m2 directory to save time. Since Maven has

# improved download parallelism, this now defaults to false.

if [[ "${PREPOPULATE_M2_REPOSITORY:-false}" == true ]] ; then

echo ">>> Populating m2 directory..."

if ! bin/jenkins/populate_m2_directory.py ; then

echo "Failed to prepopulate the m2 directory. Continuing..."

else

echo ">>> Skip populating m2 directory"

setup_postgresql

# Be careful about adding code after postgres initialization, it may fail (IMPALA-13802).

Sunbelt Computer Software

PL/B Language Development and Support

Navigation Menu

Repository navigation

bootstrap_system.sh

bootstrap_system.sh

Sunbelt Computer Software

PL/B Language Development and Support

FilesExpand file tree

bootstrap_system.sh

Latest commit

History

bootstrap_system.sh

File metadata and controls