docs: smaller documentation fixes (GitHub links, type parsing etc.) (… · idimetrix/crawlee-python@49403a1 · GitHub
Skip to content

Commit 49403a1

Browse files
barjinjanbuchar
barjin
and
janbuchar
authored
docs: smaller documentation fixes (GitHub links, type parsing etc.) (apify#640)
Fixes more issues from apify#324 . Namely: - Fixes broken GitHub links - Correctly parses parameter / return types with Python's `ast` module - Handles inheritance better (copies parent's docstring on `@override` methods) - Using `@apify/docusaurus-plugin-typedoc-api@4.2.7`, expands the `Unpack` types and further simplifies other helper types (`NotRequired`, `Optional` etc.). --------- Co-authored-by: Jan Buchar <jan.buchar@apify.com>
1 parent 0b117fd commit 49403a1

7 files changed

Lines changed: 252 additions & 171 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions

‎website/build_api_reference.sh‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,7 @@
11
#!/bin/bash
22

3-
# On macOS, sed requires a space between -i and '' to specify no backup should be done
4-
# On Linux, sed requires no space between -i and '' to specify no backup should be done
5-
sed_no_backup() {
6-
if [[ $(uname) = "Darwin" ]]; then
7-
sed -i '' "$@"
8-
else
9-
sed -i'' "$@"
10-
fi
11-
}
12-
133
# Create docspec dump of this package's source code through pydoc-markdown
144
python ./pydoc-markdown/generate_ast.py > docspec-dump.jsonl
15-
sed_no_backup "s#${PWD}/..#REPO_ROOT_PLACEHOLDER#g" docspec-dump.jsonl
165

176
rm -rf "${apify_shared_tempdir}"
187

‎website/package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"name": "crawlee",
23
"scripts": {
34
"examples": "docusaurus-examples",
45
"postinstall": "npx patch-package",
@@ -33,7 +34,7 @@
3334
"typescript": "5.6.2"
3435
},
3536
"dependencies": {
36-
"@apify/docusaurus-plugin-typedoc-api": "^4.2.6",
37+
"@apify/docusaurus-plugin-typedoc-api": "^4.2.7",
3738
"@apify/utilities": "^2.8.0",
3839
"@docusaurus/core": "^3.5.2",
3940
"@docusaurus/mdx-loader": "^3.5.2",

‎website/parse_types.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
"""
2+
Given a JSON file containing a list of expressions, this script will parse each expression and output a JSON file containing an object with the parsed expressions.
3+
Called from transformDocs.js.
4+
5+
Accepts one CLI argument: the path to the JSON file containing the expressions to parse.
6+
"""
7+
8+
import ast
9+
import json
10+
import sys
11+
import os
12+
13+
base_scalar_types = {
14+
"str",
15+
"int",
16+
"float",
17+
"bool",
18+
"bytearray",
19+
"timedelta",
20+
"None",
21+
}
22+
23+
24+
def parse_expression(ast_node, full_expression):
25+
"""
26+
Turns the AST expression object into a typedoc-compliant dict
27+
"""
28+
29+
current_node_type = ast_node.__class__.__name__
30+
31+
if current_node_type == "BinOp" and ast_node.op.__class__.__name__ == "BitOr":
32+
return {
33+
"type": "union",
34+
"types": [
35+
parse_expression(ast_node.left, full_expression),
36+
parse_expression(ast_node.right, full_expression),
37+
],
38+
}
39+
40+
if current_node_type == "Tuple":
41+
return [parse_expression(e, full_expression) for e in ast_node.elts]
42+
43+
if current_node_type == "Subscript":
44+
if "id" in ast_node.value._fields and ast_node.value.id == "Annotated":
45+
return parse_expression(ast_node.slice.dims[0], full_expression)
46+
47+
main_type = parse_expression(ast_node.value, full_expression)
48+
type_argument = parse_expression(ast_node.slice, full_expression)
49+
50+
main_type["typeArguments"] = (
51+
type_argument if isinstance(type_argument, list) else [type_argument]
52+
)
53+
return main_type
54+
55+
if current_node_type == "Constant":
56+
return {"type": "literal", "value": ast_node.value}
57+
58+
# If the expression is not one of the types above, we simply print the expression
59+
return {
60+
"type": "reference",
61+
"name": full_expression[ast_node.col_offset : ast_node.end_col_offset],
62+
}
63+
64+
65+
typedoc_types_path = sys.argv[1]
66+
67+
with open(typedoc_types_path, "r") as f:
68+
typedoc_out = {}
69+
expressions = json.load(f)
70+
71+
for expression in expressions:
72+
try:
73+
if typedoc_out.get(expression) is None:
74+
typedoc_out[expression] = parse_expression(
75+
ast.parse(expression).body[0].value, expression
76+
)
77+
except Exception as e:
78+
print(f"Invalid expression encountered while parsing: {expression}")
79+
print(f"Error: {e}")
80+
81+
with open(f"{os.path.splitext(typedoc_types_path)[0]}-parsed.json", "w") as f:
82+
f.write(json.dumps(typedoc_out, indent=4))

‎website/pydoc-markdown/generate_ast.py‎

Lines changed: 9 additions & 2 deletions

0 commit comments

Comments
 (0)