FilesExpand file tree

master

/

lambda.go

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

183 lines (155 loc) · 4.53 KB

master

/

lambda.go

Copy path

Top

File metadata and controls

• Code
• Blame

183 lines (155 loc) · 4.53 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

package corral

import (

"context"

"encoding/json"

"fmt"

"os"

"runtime/debug"

"strconv"

"sync/atomic"

log "github.com/sirupsen/logrus"

"github.com/spf13/viper"

"github.com/bcongdon/corral/internal/pkg/corfs"

"github.com/bcongdon/corral/internal/pkg/coriam"

"github.com/bcongdon/corral/internal/pkg/corlambda"

)

var (

lambdaDriver *Driver

)

// corralRoleName is the name to use when deploying an IAM role

const corralRoleName = "CorralExecutionRole"

// runningInLambda infers if the program is running in AWS lambda via inspection of the environment

func runningInLambda() bool {

expectedEnvVars := []string{"LAMBDA_TASK_ROOT", "AWS_EXECUTION_ENV", "LAMBDA_RUNTIME_DIR"}

for _, envVar := range expectedEnvVars {

if os.Getenv(envVar) == "" {

return false

}

}

return true

}

func prepareResult(job *Job) string {

result := taskResult{

BytesRead: int(job.bytesRead),

BytesWritten: int(job.bytesWritten),

}

payload, _ := json.Marshal(result)

return string(payload)

}

func handleRequest(ctx context.Context, task task) (string, error) {

// Precaution to avoid running out of memory for reused Lambdas

debug.FreeOSMemory()

// Setup current job

fs := corfs.InitFilesystem(task.FileSystemType)

currentJob := lambdaDriver.jobs[task.JobNumber]

currentJob.fileSystem = fs

currentJob.intermediateBins = task.IntermediateBins

currentJob.outputPath = task.WorkingLocation

currentJob.config.Cleanup = task.Cleanup

// Need to reset job counters in case this is a reused lambda

currentJob.bytesRead = 0

currentJob.bytesWritten = 0

if task.Phase == MapPhase {

err := currentJob.runMapper(task.BinID, task.Splits)

return prepareResult(currentJob), err

} else if task.Phase == ReducePhase {

err := currentJob.runReducer(task.BinID)

return prepareResult(currentJob), err

}

return "", fmt.Errorf("Unknown phase: %d", task.Phase)

}

type lambdaExecutor struct {

*corlambda.LambdaClient

*coriam.IAMClient

functionName string

}

func newLambdaExecutor(functionName string) *lambdaExecutor {

return &lambdaExecutor{

LambdaClient: corlambda.NewLambdaClient(),

IAMClient: coriam.NewIAMClient(),

functionName: functionName,

}

}

func loadTaskResult(payload []byte) taskResult {

// Unescape JSON string

payloadStr, _ := strconv.Unquote(string(payload))

var result taskResult

err := json.Unmarshal([]byte(payloadStr), &result)

if err != nil {

log.Errorf("%s", err)

}

return result

}

func (l *lambdaExecutor) RunMapper(job *Job, jobNumber int, binID uint, inputSplits []inputSplit) error {

mapTask := task{

JobNumber: jobNumber,

Phase: MapPhase,

BinID: binID,

Splits: inputSplits,

IntermediateBins: job.intermediateBins,

FileSystemType: corfs.S3,

WorkingLocation: job.outputPath,

}

payload, err := json.Marshal(mapTask)

if err != nil {

return err

}

resultPayload, err := l.Invoke(l.functionName, payload)

taskResult := loadTaskResult(resultPayload)

atomic.AddInt64(&job.bytesRead, int64(taskResult.BytesRead))

atomic.AddInt64(&job.bytesWritten, int64(taskResult.BytesWritten))

return err

}

func (l *lambdaExecutor) RunReducer(job *Job, jobNumber int, binID uint) error {

mapTask := task{

JobNumber: jobNumber,

Phase: ReducePhase,

BinID: binID,

FileSystemType: corfs.S3,

WorkingLocation: job.outputPath,

Cleanup: job.config.Cleanup,

}

payload, err := json.Marshal(mapTask)

if err != nil {

return err

}

resultPayload, err := l.Invoke(l.functionName, payload)

taskResult := loadTaskResult(resultPayload)

atomic.AddInt64(&job.bytesRead, int64(taskResult.BytesRead))

atomic.AddInt64(&job.bytesWritten, int64(taskResult.BytesWritten))

return err

}

func (l *lambdaExecutor) Deploy() {

var roleARN string

var err error

if viper.GetBool("lambdaManageRole") {

roleARN, err = l.DeployPermissions(corralRoleName)

if err != nil {

panic(err)

}

} else {

roleARN = viper.GetString("lambdaRoleARN")

}

config := &corlambda.FunctionConfig{

Name: l.functionName,

RoleARN: roleARN,

Timeout: viper.GetInt64("lambdaTimeout"),

MemorySize: viper.GetInt64("lambdaMemory"),

}

err = l.DeployFunction(config)

if err != nil {

panic(err)

}

}

func (l *lambdaExecutor) Undeploy() {

log.Info("Undeploying function")

err := l.LambdaClient.DeleteFunction(l.functionName)

if err != nil {

log.Errorf("Error when undeploying function: %s", err)

}

log.Info("Undeploying IAM Permissions")

err = l.IAMClient.DeletePermissions(corralRoleName)

if err != nil {

log.Errorf("Error when undeploying IAM permissions: %s", err)

}

}