FilesExpand file tree

v3.14.8

/

readline_tokenizer.c

Copy path

Blame

More file actions

Blame

More file actions

Latest commit

 

History

History

History

131 lines (124 loc) · 3.71 KB

v3.14.8

/

readline_tokenizer.c

Copy path

Top

File metadata and controls

• Code
• Blame

131 lines (124 loc) · 3.71 KB

Raw

Copy raw file

Download raw file

Open symbols panel

Edit and raw actions

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

#include "Python.h"

#include "errcode.h"

#include "helpers.h"

#include "../lexer/lexer.h"

#include "../lexer/state.h"

#include "../lexer/buffer.h"

static int

tok_readline_string(struct tok_state* tok) {

PyObject* line = NULL;

PyObject* raw_line = PyObject_CallNoArgs(tok->readline);

if (raw_line == NULL) {

if (PyErr_ExceptionMatches(PyExc_StopIteration)) {

PyErr_Clear();

return 1;

}

_PyTokenizer_error_ret(tok);

goto error;

}

if(tok->encoding != NULL) {

if (!PyBytes_Check(raw_line)) {

PyErr_Format(PyExc_TypeError, "readline() returned a non-bytes object");

_PyTokenizer_error_ret(tok);

goto error;

}

line = PyUnicode_Decode(PyBytes_AS_STRING(raw_line), PyBytes_GET_SIZE(raw_line),

tok->encoding, "replace");

Py_CLEAR(raw_line);

if (line == NULL) {

_PyTokenizer_error_ret(tok);

goto error;

}

} else {

if(!PyUnicode_Check(raw_line)) {

PyErr_Format(PyExc_TypeError, "readline() returned a non-string object");

_PyTokenizer_error_ret(tok);

goto error;

}

line = raw_line;

raw_line = NULL;

}

Py_ssize_t buflen;

const char* buf = PyUnicode_AsUTF8AndSize(line, &buflen);

if (buf == NULL) {

_PyTokenizer_error_ret(tok);

goto error;

}

// Make room for the null terminator *and* potentially

// an extra newline character that we may need to artificially

// add.

size_t buffer_size = buflen + 2;

if (!_PyLexer_tok_reserve_buf(tok, buffer_size)) {

goto error;

}

memcpy(tok->inp, buf, buflen);

tok->inp += buflen;

*tok->inp = '\0';

tok->line_start = tok->cur;

Py_DECREF(line);

return 1;

error:

Py_XDECREF(raw_line);

Py_XDECREF(line);

return 0;

}

static int

tok_underflow_readline(struct tok_state* tok) {

assert(tok->decoding_state == STATE_NORMAL);

assert(tok->fp == NULL && tok->input == NULL && tok->decoding_readline == NULL);

if (tok->start == NULL && !INSIDE_FSTRING(tok)) {

tok->cur = tok->inp = tok->buf;

}

if (!tok_readline_string(tok)) {

return 0;

}

if (tok->inp == tok->cur) {

tok->done = E_EOF;

return 0;

}

tok->implicit_newline = 0;

if (tok->inp[-1] != '\n') {

assert(tok->inp + 1 < tok->end);

/* Last line does not end in \n, fake one */

*tok->inp++ = '\n';

*tok->inp = '\0';

tok->implicit_newline = 1;

}

ADVANCE_LINENO();

/* The default encoding is UTF-8, so make sure we don't have any

non-UTF-8 sequences in it. */

if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) {

_PyTokenizer_error_ret(tok);

return 0;

}

assert(tok->done == E_OK);

return tok->done == E_OK;

}

struct tok_state *

_PyTokenizer_FromReadline(PyObject* readline, const char* enc,

int exec_input, int preserve_crlf)

{

struct tok_state *tok = _PyTokenizer_tok_new();

if (tok == NULL)

return NULL;

if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {

_PyTokenizer_Free(tok);

PyErr_NoMemory();

return NULL;

}

tok->cur = tok->inp = tok->buf;

tok->end = tok->buf + BUFSIZ;

tok->fp = NULL;

if (enc != NULL) {

tok->encoding = _PyTokenizer_new_string(enc, strlen(enc), tok);

if (!tok->encoding) {

_PyTokenizer_Free(tok);

return NULL;

}

}

tok->decoding_state = STATE_NORMAL;

tok->underflow = &tok_underflow_readline;

Py_INCREF(readline);

tok->readline = readline;

return tok;

}