Basic repository setup (#1) · polya20/crawlee-python@421ee8e · GitHub
Skip to content

Commit 421ee8e

Browse files
vdusekvdusek
authored
Basic repository setup (apify#1)
1 parent 3cd56e2 commit 421ee8e

27 files changed

Lines changed: 1039 additions & 0 deletions

‎.editorconfig‎

Lines changed: 15 additions & 0 deletions

‎.github/CODEOWNERS‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Documentation codeowner
2+
3+
/docs/*.md @TC-MO
4+
/docs/*.mdx @TC-MO

‎.github/workflows/check_version_availability.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Check package version availability
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
check_version_availability:
8+
name: Check version availability
9+
runs-on: ubuntu-latest
10+
if: (!startsWith(github.event.pull_request.title, 'docs:'))
11+
12+
steps:
13+
# We need to check out the head commit in case of PRs, and the default ref otherwise (during release).
14+
- name: Checkout repository
15+
uses: actions/checkout@v4
16+
with:
17+
ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}"
18+
19+
- name: Set up Python
20+
uses: actions/setup-python@v4
21+
with:
22+
python-version: "3.8"
23+
24+
- name: Install dependencies
25+
run: make install-dev
26+
27+
- name: Check version availability
28+
run: make check-version-availability

‎.github/workflows/docs.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# TODO

‎.github/workflows/integration_tests.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# TODO

‎.github/workflows/lint_and_type_checks.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Lint and type checks
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
lint_and_type_checks:
8+
name: Lint and type checks
9+
runs-on: ubuntu-latest
10+
strategy:
11+
matrix:
12+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
13+
14+
steps:
15+
# We need to check out the head commit in case of PRs, and the default ref otherwise (during release).
16+
- name: Checkout repository
17+
uses: actions/checkout@v4
18+
with:
19+
ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}"
20+
21+
- name: Set up Python ${{ matrix.python-version }}
22+
uses: actions/setup-python@v4
23+
with:
24+
python-version: ${{ matrix.python-version }}
25+
26+
- name: Install dependencies
27+
run: make install-dev
28+
29+
- name: Run lint
30+
run: make lint
31+
32+
- name: Run type checks
33+
run: make type-check

‎.github/workflows/release.yaml‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
name: Check & Release
2+
3+
on:
4+
# Push to master will publish a beta version
5+
push:
6+
branches:
7+
- master
8+
tags-ignore:
9+
- "**"
10+
# A release via GitHub releases will publish a stable version
11+
release:
12+
types: [published]
13+
# Workflow dispatch will publish whatever you choose
14+
workflow_dispatch:
15+
inputs:
16+
release_type:
17+
description: Release type
18+
required: true
19+
type: choice
20+
default: alpha
21+
options:
22+
- alpha
23+
- beta
24+
- final
25+
26+
jobs:
27+
should_release:
28+
name: Check whether to release
29+
if: (!startsWith(github.event.head_commit.message, 'docs:') || github.event_name == 'workflow_dispatch')
30+
runs-on: ubuntu-latest
31+
steps:
32+
- name: Dummy step
33+
run: true
34+
35+
lint_and_type_checks:
36+
name: Run lint and type_checks
37+
needs: [should_release]
38+
uses: ./.github/workflows/lint_and_type_checks.yaml
39+
40+
unit_tests:
41+
name: Run unit tests
42+
needs: [should_release]
43+
uses: ./.github/workflows/unit_tests.yaml
44+
45+
integration_tests:
46+
name: Run integration tests
47+
needs: [should_release]
48+
uses: ./.github/workflows/integration_tests.yaml
49+
secrets: inherit
50+
51+
publish_to_pypi:
52+
name: Publish to PyPI
53+
needs: [should_release, lint_and_type_checks, unit_tests, integration_tests]
54+
runs-on: ubuntu-latest
55+
permissions:
56+
contents: write
57+
id-token: write
58+
environment:
59+
name: pypi
60+
url: https://pypi.org/p/apify
61+
62+
steps:
63+
- name: Checkout repository
64+
uses: actions/checkout@v4
65+
66+
- name: Set up Python
67+
uses: actions/setup-python@v4
68+
with:
69+
python-version: 3.8
70+
71+
- name: Install dependencies
72+
run: make install-dev
73+
74+
- # Determine if this is a prerelease or latest release
75+
name: Determine release type
76+
id: get-release-type
77+
run: |
78+
if [ ${{ github.event_name }} = release ]; then
79+
release_type="final"
80+
elif [ ${{ github.event_name }} = push ]; then
81+
release_type="beta"
82+
elif [ ${{ github.event_name }} = workflow_dispatch ]; then
83+
release_type=${{ github.event.inputs.release_type }}
84+
fi
85+
86+
if [ ${release_type} = final ]; then
87+
docker_image_tag="latest"
88+
elif [ ${release_type} = beta ]; then
89+
docker_image_tag="beta"
90+
else
91+
docker_image_tag=""
92+
fi
93+
94+
echo "release_type=${release_type}" >> $GITHUB_OUTPUT
95+
echo "docker_image_tag=${docker_image_tag}" >> $GITHUB_OUTPUT
96+
97+
- # Check whether the released version is listed in CHANGELOG.md
98+
name: Check whether the released version is listed in the changelog
99+
if: steps.get-release-type.outputs.release_type != 'alpha'
100+
run: make check-changelog-entry
101+
102+
- # Check version consistency and increment pre-release version number for prereleases (must be the last step before build)
103+
name: Bump pre-release version
104+
if: steps.get-release-type.outputs.release_type != 'final'
105+
run: python ./scripts/update_version_for_prerelease.py ${{ steps.get-release-type.outputs.release_type }}
106+
107+
- # Build a source distribution and a python3-only wheel
108+
name: Build distribution files
109+
run: make build
110+
111+
- # Check whether the package description will render correctly on PyPI
112+
name: Check package rendering on PyPI
113+
run: make twine-check
114+
115+
- # Publish package to PyPI using their official GitHub action
116+
name: Publish package to PyPI
117+
uses: pypa/gh-action-pypi-publish@release/v1
118+
119+
- # Tag the current commit with the version tag if this is not made from the release event (releases are tagged with the release process)
120+
name: Tag Version
121+
if: github.event_name != 'release'
122+
run: |
123+
git_tag=v`python ./scripts/print_current_package_version.py`
124+
git tag $git_tag
125+
git push origin $git_tag
126+
127+
- # Upload the build artifacts to the release
128+
name: Upload the build artifacts to release
129+
if: github.event_name == 'release'
130+
run: gh release upload ${{ github.ref_name }} dist/*
131+
env:
132+
GH_TOKEN: ${{ github.token }}
133+
134+
- # Trigger building the Python Docker images in apify/apify-actor-docker repo
135+
name: Trigger Docker image build
136+
run: |
137+
PACKAGE_VERSION=`python ./scripts/print_current_package_version.py`
138+
gh api -X POST "/repos/apify/apify-actor-docker/dispatches" \
139+
-F event_type=build-python-images \
140+
-F client_payload[release_tag]=${{ steps.get-release-type.outputs.docker_image_tag }} \
141+
-F client_payload[apify_version]=$PACKAGE_VERSION
142+
env:
143+
GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}

‎.github/workflows/run_checks.yaml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Code quality checks
2+
3+
on:
4+
pull_request_target:
5+
6+
jobs:
7+
check_version_availability:
8+
name: Check version availability
9+
uses: ./.github/workflows/check_version_availability.yaml
10+
11+
lint_and_type_checks:
12+
name: Run lint and type checks
13+
uses: ./.github/workflows/lint_and_type_checks.yaml
14+
15+
unit_tests:
16+
name: Run unit tests
17+
needs: [lint_and_type_checks]
18+
uses: ./.github/workflows/unit_tests.yaml
19+
20+
# If the PR comes from the main repo, run integration tests directly
21+
integration_tests:
22+
if: github.event.pull_request.head.repo.owner.login == 'apify'
23+
name: Run integration tests
24+
needs: [lint_and_type_checks, unit_tests]
25+
uses: ./.github/workflows/integration_tests.yaml
26+
secrets: inherit
27+
28+
# If the PR comes from a fork,
29+
# we need to approve running the workflow first before allowing it to run,
30+
# so that we can check for any unauthorized access to our secrets.
31+
# We need two workflow jobs for that,
32+
# because jobs calling reusable workflows can't use an environment.
33+
# The first job is a dummy job that just asks for approval to use the `fork-worklows` environment.
34+
integration_tests_fork_approve:
35+
if: github.event.pull_request.head.repo.owner.login != 'apify'
36+
name: Approve integration tests from fork
37+
needs: [lint_and_type_checks, unit_tests]
38+
environment: fork-pr-integration-tests
39+
runs-on: ubuntu-latest
40+
steps:
41+
- name: Dummy step
42+
run: true
43+
44+
# The second job is the actual integration tests job.
45+
integration_tests_fork:
46+
name: Run integration tests from fork
47+
needs: [integration_tests_fork_approve]
48+
uses: ./.github/workflows/integration_tests.yaml
49+
secrets: inherit

‎.github/workflows/unit_tests.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Unit tests
2+
3+
on:
4+
workflow_call:
5+
6+
jobs:
7+
unit_tests:
8+
name: Run unit tests
9+
strategy:
10+
matrix:
11+
os: [ubuntu-latest, windows-latest]
12+
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
13+
runs-on: ${{ matrix.os }}
14+
15+
steps:
16+
# We need to check out the head commit in case of PRs,
17+
# and the default ref otherwise (during release).
18+
- name: Checkout repository
19+
uses: actions/checkout@v4
20+
with:
21+
ref: "${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}"
22+
23+
- name: Set up Python ${{ matrix.python-version }}
24+
uses: actions/setup-python@v4
25+
with:
26+
python-version: ${{ matrix.python-version }}
27+
28+
- name: Install dependencies
29+
run: make install-dev
30+
31+
- name: Run unit tests
32+
run: make unit-tests

‎.gitignore‎

Lines changed: 22 additions & 0 deletions

0 commit comments

Comments
 (0)