BuildDir() string

CacheDir() string

DepDir() string

DepsIdx() string

LinkDirectoryInDepDir(string, string) error

WriteEnvFile(string, string) error

WriteProfileD(string, string) error

AllDependencyVersions(string) []string

DefaultVersion(string) (libbuildpack.Dependency, error)

IsCached() bool

Execute(string, io.Writer, io.Writer, string, ...string) error

Output(dir string, program string, args ...string) (string, error)

RunWithOutput(*exec.Cmd) ([]byte, error)

PythonVersion string

Manifest Manifest

Installer Installer

Stager Stager

Command Command

Log *libbuildpack.Logger

Logfile *os.File

HasNltkData bool

removeRequirementsText bool

if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "environment.yml")); err != nil {

s.Log.Error("Error checking existence of environment.yml: %v", err)

return err

} else if exists {

return conda.Run(conda.New(s.Installer, s.Stager, s.Command, s.Log))

} else {

return RunPython(s)

}

s.Log.BeginStep("Supplying Python")

dirSnapshot := snapshot.Dir(s.Stager.BuildDir(), s.Log)

if err := s.SetupCacheDir(); err != nil {

s.Log.Error("Error setting up cache: %v", err)

return err

}

if err := s.CopyRuntimeTxt(); err != nil {

s.Log.Error("Error copying runtime.txt to deps dir: %v", err)

return err

}

if err := s.HandlePipfile(); err != nil {

s.Log.Error("Error checking for Pipfile.lock: %v", err)

return err

}

if err := s.InstallPython(); err != nil {

s.Log.Error("Could not install python: %v", err)

return err

}

if err := s.InstallPipPop(); err != nil {

s.Log.Error("Could not install pip pop: %v", err)

return err

}

if err := s.InstallPipEnv(); err != nil {

s.Log.Error("Could not install pipenv: %v", err)

return err

}

if err := s.HandleRequirementstxt(); err != nil {

s.Log.Error("Error checking requirements.txt: %v", err)

return err

}

if err := s.HandlePylibmc(); err != nil {

s.Log.Error("Error checking Pylibmc: %v", err)

return err

}

if err := s.HandleFfi(); err != nil {

s.Log.Error("Error checking ffi: %v", err)

return err

}

if err := s.HandleMercurial(); err != nil {

s.Log.Error("Could not handle pip mercurial dependencies: %v", err)

return err

}

if err := s.UninstallUnusedDependencies(); err != nil {

s.Log.Error("Error uninstalling unused dependencies: %v", err)

return err

}

if err := s.RunPip(); err != nil {

s.Log.Error("Could not install pip packages: %v", err)

return err

}

if err := s.DownloadNLTKCorpora(); err != nil {

s.Log.Error("Could not download NLTK Corpora: %v", err)

return err

}

if err := s.RewriteShebangs(); err != nil {

s.Log.Error("Unable to rewrite she-bangs: %s", err.Error())

return err

}

if err := s.CreateDefaultEnv(); err != nil {

s.Log.Error("Unable to setup default environment: %s", err.Error())

return err

}

if cacheDirSize, err := s.Command.Output(os.Getenv("XDG_CACHE_HOME"), "du", "--summarize", os.Getenv("XDG_CACHE_HOME")); err == nil {

s.Log.Debug("Size of pip cache dir: %s", cacheDirSize)

}

if s.removeRequirementsText {

if err := os.Remove(filepath.Join(s.Stager.BuildDir(), "requirements.txt")); err != nil {

s.Log.Error("Unable to clean up app directory: %s", err.Error())

return err

}

}

dirSnapshot.Diff()

return nil

if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "runtime.txt")); err != nil {

return err

} else if exists {

if err = libbuildpack.CopyFile(filepath.Join(s.Stager.BuildDir(), "runtime.txt"), filepath.Join(s.Stager.DepDir(), "runtime.txt")); err != nil {

return err

}

}

return nil

if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "grep", "-Fiq", "hg+", "requirements.txt"); err != nil {

return nil

}

if s.Manifest.IsCached() {

s.Log.Warning("Cloud Foundry does not support Pip Mercurial dependencies while in offline-mode. Vendor your dependencies if they do not work.")

}

if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "python", "-m", "pip", "install", "mercurial"); err != nil {

return err

}

if err := s.Stager.LinkDirectoryInDepDir(filepath.Join(s.Stager.DepDir(), "python", "bin"), "bin"); err != nil {

return err

}

return nil

var pipfileExists, runtimeExists bool

var pipfileJson pipfile.Lock

var err error

if pipfileExists, err = libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "Pipfile.lock")); err != nil {

return err

}

if runtimeExists, err = libbuildpack.FileExists(filepath.Join(s.Stager.DepDir(), "runtime.txt")); err != nil {

return err

}

if pipfileExists && !runtimeExists {

if err = libbuildpack.NewJSON().Load(filepath.Join(s.Stager.BuildDir(), "Pipfile.lock"), &pipfileJson); err != nil {

return err

}

formattedVersion := s.formatVersion(pipfileJson.Meta.Requires.Version)

if err := ioutil.WriteFile(filepath.Join(s.Stager.DepDir(), "runtime.txt"), []byte(formattedVersion), 0644); err != nil {

return err

}

}

return nil

var dep libbuildpack.Dependency

runtimetxtExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.DepDir(), "runtime.txt"))

if err != nil {

return err

}

if runtimetxtExists {

userDefinedVersion, err := ioutil.ReadFile(filepath.Join(s.Stager.DepDir(), "runtime.txt"))

if err != nil {

return err

}

s.PythonVersion = strings.TrimSpace(strings.NewReplacer("\\r", "", "\\n", "").Replace(string(userDefinedVersion)))

s.Log.Debug("***Version info: (%s)", s.PythonVersion)

}

if s.PythonVersion != "" {

versions := s.Manifest.AllDependencyVersions("python")

shortPythonVersion := strings.TrimLeft(s.PythonVersion, "python-")

s.Log.Debug("***Version info: (%s) (%s)", s.PythonVersion, shortPythonVersion)

ver, err := libbuildpack.FindMatchingVersion(shortPythonVersion, versions)

if err != nil {

return err

}

dep.Name = "python"

dep.Version = ver

s.Log.Debug("***Version info: %s, %s, %s", dep.Name, s.PythonVersion, dep.Version)

} else {

var err error

dep, err = s.Manifest.DefaultVersion("python")

if err != nil {

return err

}

}

pythonInstallDir := filepath.Join(s.Stager.DepDir(), "python")

if err := s.Installer.InstallDependency(dep, pythonInstallDir); err != nil {

return err

}

if err := s.Stager.LinkDirectoryInDepDir(filepath.Join(pythonInstallDir, "bin"), "bin"); err != nil {

return err

}

if found, err := libbuildpack.FileExists(filepath.Join(pythonInstallDir, "usr", "lib", "x86_64-linux-gnu")); err != nil {

return err

} else if found {

if err := s.Stager.LinkDirectoryInDepDir(filepath.Join(pythonInstallDir, "usr", "lib", "x86_64-linux-gnu"), "lib"); err != nil {

return err

}

}

if err := s.Stager.LinkDirectoryInDepDir(filepath.Join(pythonInstallDir, "lib"), "lib"); err != nil {

return err

}

if err := os.Setenv("PATH", fmt.Sprintf("%s:%s", filepath.Join(s.Stager.DepDir(), "bin"), os.Getenv("PATH"))); err != nil {

return err

}

if err := os.Setenv("PYTHONPATH", filepath.Join(s.Stager.DepDir())); err != nil {

return err

}

return nil

files, err := filepath.Glob(filepath.Join(s.Stager.DepDir(), "bin", "*"))

if err != nil {

return err

}

for _, file := range files {

if fileInfo, err := os.Stat(file); err != nil {

return err

} else if fileInfo.IsDir() {

continue

}

fileContents, err := ioutil.ReadFile(file)

if err != nil {

return err

}

shebangRegex := regexp.MustCompile(`^#!/.*/python.*`)

fileContents = shebangRegex.ReplaceAll(fileContents, []byte("#!/usr/bin/env python"))

if err := ioutil.WriteFile(file, fileContents, 0755); err != nil {

return err

}

}

return nil

tempPath := filepath.Join("/tmp", "pip-pop")

if err := s.Installer.InstallOnlyVersion("pip-pop", tempPath); err != nil {

return err

}

if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "python", "-m", "pip", "install", "pip-pop", "--exists-action=w", "--no-index", fmt.Sprintf("--find-links=%s", tempPath)); err != nil {

s.Log.Debug("******Path val: %s", os.Getenv("PATH"))

return err

}

if err := s.Stager.LinkDirectoryInDepDir(filepath.Join(s.Stager.DepDir(), "python", "bin"), "bin"); err != nil {

return err

}

return nil

requirementstxtExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "requirements.txt"))

if err != nil {

return err

} else if requirementstxtExists {

return nil

}

pipfileExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "Pipfile"))

if err != nil {

return err

} else if !pipfileExists {

return nil

}

hasLockFile, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "Pipfile.lock"))

if err != nil {

return fmt.Errorf("could not check Pipfile.lock existence: %v", err)

} else if hasLockFile {

s.Log.Info("Generating 'requirements.txt' from Pipfile.lock")

requirementsContents, err := pipfileToRequirements(filepath.Join(s.Stager.BuildDir(), "Pipfile.lock"))

if err != nil {

return fmt.Errorf("failed to write `requirement.txt` from Pipfile.lock: %s", err.Error())

}

return s.writeTempRequirementsTxt(requirementsContents)

}

s.Log.Info("Installing pipenv")

if err := s.Installer.InstallOnlyVersion("pipenv", filepath.Join("/tmp", "pipenv")); err != nil {

return err

}

if err := s.installFfi(); err != nil {

return err

}

for _, dep := range []string{"setuptools_scm", "pytest-runner", "parver", "invoke", "pipenv", "wheel"} {

s.Log.Info("Installing %s", dep)

out := &bytes.Buffer{}

stderr := &bytes.Buffer{}

if err := s.Command.Execute(s.Stager.BuildDir(), out, stderr, "python", "-m", "pip", "install", dep, "--exists-action=w", "--no-index", fmt.Sprintf("--find-links=%s", filepath.Join("/tmp", "pipenv"))); err != nil {

return fmt.Errorf("Failed to install %s: %v.\nStdout: %v\nStderr: %v", dep, err, out, stderr)

}

}

s.Stager.LinkDirectoryInDepDir(filepath.Join(s.Stager.DepDir(), "python", "bin"), "bin")

s.Log.Info("Generating 'requirements.txt' with pipenv")

cmd := exec.Command("pipenv", "lock", "--requirements")

cmd.Dir = s.Stager.BuildDir()

cmd.Env = append(os.Environ(), "VIRTUALENV_NEVER_DOWNLOAD=true")

output, err := s.Command.RunWithOutput(cmd)

if err != nil {

return err

}

outputString := string(output)

// Remove output due to virtualenv

if strings.HasPrefix(outputString, "Using ") {

reqs := strings.SplitN(outputString, "\n", 2)

if len(reqs) > 0 {

outputString = reqs[1]

}

}

return s.writeTempRequirementsTxt(outputString)

var lockFile struct {

Meta struct {

Sources []struct {

URL string

}

} `json:"_meta"`

Default map[string]struct {

Version string

}

}

lockContents, err := ioutil.ReadFile(lockFilePath)

if err != nil {

return "", err

}

err = json.Unmarshal(lockContents, &lockFile)

if err != nil {

return "", err

}

buf := &bytes.Buffer{}

for _, source := range lockFile.Meta.Sources {

fmt.Fprintf(buf, "-i %s\n", source.URL)

}

for pkg, obj := range lockFile.Default {

fmt.Fprintf(buf, "%s%s\n", pkg, obj.Version)

}

return buf.String(), nil

memcachedDir := filepath.Join(s.Stager.DepDir(), "libmemcache")

if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "pip-grep", "-s", "requirements.txt", "pylibmc"); err == nil {

s.Log.BeginStep("Noticed pylibmc. Bootstrapping libmemcached.")

if err := s.Installer.InstallOnlyVersion("libmemcache", memcachedDir); err != nil {

return err

}

os.Setenv("LIBMEMCACHED", memcachedDir)

s.Stager.WriteEnvFile("LIBMEMCACHED", memcachedDir)

s.Stager.LinkDirectoryInDepDir(filepath.Join(memcachedDir, "lib"), "lib")

s.Stager.LinkDirectoryInDepDir(filepath.Join(memcachedDir, "lib", "sasl2"), "lib")

s.Stager.LinkDirectoryInDepDir(filepath.Join(memcachedDir, "lib", "pkgconfig"), "pkgconfig")

s.Stager.LinkDirectoryInDepDir(filepath.Join(memcachedDir, "include"), "include")

}

return nil

if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "requirements.txt")); err != nil {

return err

} else if exists {

return nil

}

if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "setup.py")); err != nil {

return err

} else if !exists {

return nil

}

return s.writeTempRequirementsTxt("-e .")

ffiDir := filepath.Join(s.Stager.DepDir(), "libffi")

// Only install libffi if we haven't done so already

// This could be installed twice because pipenv installs it, but

// we later run HandleFfi, which installs it if a dependency

// from requirements.txt needs libffi.

if os.Getenv("LIBFFI") != ffiDir {

s.Log.BeginStep("Noticed dependency requiring libffi. Bootstrapping libffi.")

if err := s.Installer.InstallOnlyVersion("libffi", ffiDir); err != nil {

return err

}

versions := s.Manifest.AllDependencyVersions("libffi")

os.Setenv("LIBFFI", ffiDir)

s.Stager.WriteEnvFile("LIBFFI", ffiDir)

s.Stager.LinkDirectoryInDepDir(filepath.Join(ffiDir, "lib"), "lib")

s.Stager.LinkDirectoryInDepDir(filepath.Join(ffiDir, "lib", "pkgconfig"), "pkgconfig")

s.Stager.LinkDirectoryInDepDir(filepath.Join(ffiDir, "lib", "libffi-"+versions[0], "include"), "include")

}

return nil

if err := s.Command.Execute(s.Stager.BuildDir(), ioutil.Discard, ioutil.Discard, "pip-grep", "-s", "requirements.txt", "pymysql", "argon2-cffi", "bcrypt", "cffi", "cryptography", "django[argon2]", "Django[argon2]", "django[bcrypt]", "Django[bcrypt]", "PyNaCl", "pyOpenSSL", "PyOpenSSL", "requests[security]", "misaka"); err == nil {

return s.installFfi()

}

return nil

requirementsDeclaredExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.DepDir(), "python", "requirements-declared.txt"))

if err != nil {

return err

}

if requirementsDeclaredExists {

fileContents, _ := ioutil.ReadFile(filepath.Join(s.Stager.DepDir(), "python", "requirements-declared.txt"))

s.Log.Info("requirements-declared: %s", string(fileContents))

staleContents, err := s.Command.Output(

s.Stager.BuildDir(),

"pip-diff",

"--stale",

filepath.Join(s.Stager.DepDir(), "python", "requirements-declared.txt"),

filepath.Join(s.Stager.BuildDir(), "requirements.txt"),

"--exclude",

"setuptools",

"pip",

"wheel",

)

if err != nil {

return err

}

if staleContents == "" {

return nil

}

if err := ioutil.WriteFile(filepath.Join(s.Stager.DepDir(), "python", "requirements-stale.txt"), []byte(staleContents), 0644); err != nil {

return err

}

s.Log.BeginStep("Uninstalling stale dependencies")

if err := s.Command.Execute(

s.Stager.BuildDir(),

indentWriter(os.Stdout),

indentWriter(os.Stderr),

"python",

"-m",

"pip",

"uninstall",

"-r",

filepath.Join(s.Stager.DepDir(), "python", "requirements-stale.txt", "-y", "--exists-action=w"),

); err != nil {

return err

}

}

return nil

s.Log.BeginStep("Running Pip Install")

if os.Getenv("PIP_CERT") == "" {

os.Setenv("PIP_CERT", "/etc/ssl/certs/ca-certificates.crt")

}

requirementsPath := filepath.Join(s.Stager.BuildDir(), "requirements.txt")

if exists, err := libbuildpack.FileExists(requirementsPath); err != nil {

return fmt.Errorf("could not determine existence of requirements.txt: %v", err)

} else if !exists {

s.Log.Debug("Skipping 'pip install' since requirements.txt does not exist")

return nil

}

vendorExists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "vendor"))

if err != nil {

return fmt.Errorf("could not check vendor existence: %v", err)

}

installArgs := []string{"-m", "pip", "install", "-r", requirementsPath, "--ignore-installed", "--exists-action=w", "--src=" + filepath.Join(s.Stager.DepDir(), "src")}

var originalReqs []byte

if vendorExists {

installArgs = append(installArgs, "--no-index", "--find-links=file://"+filepath.Join(s.Stager.BuildDir(), "vendor"))

// Remove lines from requirements.txt that begin with -i

// because specifying index links here makes pip always want internet

// and pipenv generates requirements.txt with -i

originalReqs, err = ioutil.ReadFile(requirementsPath)

if err != nil {

return fmt.Errorf("could not read requirements.txt: %v", err)

}

re := regexp.MustCompile("(?m)^\\W*-i.*$")

modifiedReqs := re.ReplaceAll(originalReqs, []byte{})

err = ioutil.WriteFile(requirementsPath, modifiedReqs, 0644)

if err != nil {

return fmt.Errorf("could not overwrite requirements file: %v", err)

}

}

if err := s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "python", installArgs...); err != nil {

if vendorExists {

s.Log.Info("Running pip install without indexes failed. Not all dependencies were vendored. Trying again with indexes.")

if err := ioutil.WriteFile(requirementsPath, originalReqs, 0644); err != nil {

return fmt.Errorf("could not overwrite modified requirements file: %v", err)

}

if err = s.Command.Execute(s.Stager.BuildDir(), indentWriter(os.Stdout), indentWriter(os.Stderr), "python", installArgs...); err != nil {

s.Log.Info("Running pip install failed. You need to include all dependencies in the vendor directory.")

return fmt.Errorf("could not run pip: %v", err)

}

} else {

return fmt.Errorf("could not run pip: %v", err)

}

}

return s.Stager.LinkDirectoryInDepDir(filepath.Join(s.Stager.DepDir(), "python", "bin"), "bin")

var environmentVars = map[string]string{

"PYTHONPATH": s.Stager.DepDir(),

"LIBRARY_PATH": filepath.Join(s.Stager.DepDir(), "lib"),

"PYTHONHOME": filepath.Join(s.Stager.DepDir(), "python"),

"PYTHONUNBUFFERED": "1",

"PYTHONHASHSEED": "random",

"LANG": "en_US.UTF-8",

}

scriptContents := fmt.Sprintf(`export LANG=${LANG:-en_US.UTF-8}

`, s.Stager.DepsIdx(), s.Stager.DepsIdx())

if s.HasNltkData {

scriptContents += fmt.Sprintf(`export NLTK_DATA=$DEPS_DIR/%s/python/nltk_data`, s.Stager.DepsIdx())

environmentVars["NLTK_DATA"] = filepath.Join(s.Stager.DepDir(), "python", "nltk_data")

}

for envVar, envValue := range environmentVars {

if err := s.Stager.WriteEnvFile(envVar, envValue); err != nil {

return err

}

}

return s.Stager.WriteProfileD("python.sh", scriptContents)

if err := s.Command.Execute("/", ioutil.Discard, ioutil.Discard, "python", "-m", "nltk.downloader", "-h"); err != nil {

return nil

}

s.Log.BeginStep("Downloading NLTK corpora...")

if exists, err := libbuildpack.FileExists(filepath.Join(s.Stager.BuildDir(), "nltk.txt")); err != nil {

return fmt.Errorf("Couldn't check nltk.txt existence: %v", err)

} else if !exists {

s.Log.Info("nltk.txt not found, not downloading any corpora")

return nil

}

bPackages, err := ioutil.ReadFile(filepath.Join(s.Stager.BuildDir(), "nltk.txt"))

if err != nil {

return err

}

sPackages := strings.TrimSpace(strings.NewReplacer("\r", " ", "\n", " ").Replace(string(bPackages)))

args := []string{"-m", "nltk.downloader", "-d", filepath.Join(s.Stager.DepDir(), "python", "nltk_data")}

args = append(args, strings.Split(sPackages, " ")...)

s.Log.BeginStep("Downloading NLTK packages: %s", sPackages)

if err := s.Command.Execute("/", indentWriter(os.Stdout), indentWriter(os.Stderr), "python", args...); err != nil {

return err

}

s.HasNltkData = true

return nil

verSlice := strings.Split(version, ".")

if len(verSlice) < 3 {

return fmt.Sprintf("python-%s.x", version)

}

return fmt.Sprintf("python-%s", version)

if err := os.Setenv("XDG_CACHE_HOME", filepath.Join(s.Stager.CacheDir(), "pip_cache")); err != nil {

return err

}

if err := s.Stager.WriteEnvFile("XDG_CACHE_HOME", filepath.Join(s.Stager.CacheDir(), "pip_cache")); err != nil {

return err

}

return nil